mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into litellm_callbacks_next
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> # Conflicts: # litellm-rust/Cargo.lock # litellm-rust/Cargo.toml # litellm-rust/crates/callbacks-legacy-python/src/logger.rs # litellm-rust/crates/python-bridge/AGENTS.md # litellm-rust/crates/python-bridge/Cargo.toml # litellm-rust/crates/python-bridge/src/routes/messages/mod.rs # litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs # tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py
This commit is contained in:
commit
9649b7455b
124 changed files with 10994 additions and 853 deletions
|
|
@ -20,6 +20,7 @@ from typing import (
|
|||
)
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
)
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
|
||||
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
|
||||
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
|
||||
from litellm.llms.base_llm.managed_resources.isolation import (
|
||||
build_list_page,
|
||||
|
|
@ -59,6 +61,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
get_content_type_from_file_object,
|
||||
get_model_id_from_unified_batch_id,
|
||||
get_original_file_id,
|
||||
is_litellm_executed_batch,
|
||||
map_raw_file_ids_to_unified,
|
||||
normalize_mime_type_for_provider,
|
||||
resolve_managed_output_file_model_name,
|
||||
|
|
@ -75,6 +78,7 @@ from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccess
|
|||
CreateFileRequest,
|
||||
FileListPage,
|
||||
FileObject,
|
||||
HttpxBinaryResponseContent,
|
||||
OpenAIFileObject,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
|
|
@ -86,10 +90,6 @@ from litellm.types.utils import (
|
|||
SpecialEnums,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
from prisma.models import (
|
||||
|
|
@ -204,6 +204,19 @@ def _managed_object_table(prisma_client: PrismaClient) -> _ManagedObjectTableAct
|
|||
return prisma_client.db.litellm_managedobjecttable
|
||||
|
||||
|
||||
def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, str]:
|
||||
hidden_params: Final = cast( # cast-ok: _hidden_params is an untyped attribute the upload path sets
|
||||
"Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {}
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key in ("storage_backend", "storage_url")
|
||||
if isinstance(value := hidden_params.get(key), str)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
||||
# Class variables or attributes
|
||||
def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
|
||||
|
|
@ -226,6 +239,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> None:
|
||||
verbose_logger.info(f"Storing LiteLLM Managed File object with id={file_id} in cache")
|
||||
storage_metadata: Final = _storage_metadata_of(file_object)
|
||||
if file_object is not None:
|
||||
litellm_managed_file_object = LiteLLM_ManagedFileTable(
|
||||
unified_file_id=file_id,
|
||||
|
|
@ -235,6 +249,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
created_by=resolve_resource_owner_id(user_api_key_dict),
|
||||
team_id=user_api_key_dict.team_id,
|
||||
updated_by=user_api_key_dict.user_id,
|
||||
storage_backend=storage_metadata.get("storage_backend"),
|
||||
storage_url=storage_metadata.get("storage_url"),
|
||||
)
|
||||
await self.internal_usage_cache.async_set_cache(
|
||||
key=file_id,
|
||||
|
|
@ -262,14 +278,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
file_object_json = file_object.model_dump_json()
|
||||
db_data["file_object"] = file_object_json
|
||||
update_data["file_object"] = file_object_json
|
||||
# Extract storage metadata from hidden params if present
|
||||
hidden_params = getattr(file_object, "_hidden_params", {}) or {}
|
||||
if "storage_backend" in hidden_params:
|
||||
db_data["storage_backend"] = hidden_params["storage_backend"]
|
||||
update_data["storage_backend"] = hidden_params["storage_backend"]
|
||||
if "storage_url" in hidden_params:
|
||||
db_data["storage_url"] = hidden_params["storage_url"]
|
||||
update_data["storage_url"] = hidden_params["storage_url"]
|
||||
db_data.update(storage_metadata)
|
||||
update_data.update(storage_metadata)
|
||||
|
||||
verbose_logger.debug(
|
||||
f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
|
||||
|
|
@ -314,6 +324,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
request_tags: Sequence[str] | None = None,
|
||||
persist_attribution: bool = False,
|
||||
create_if_missing: bool = True,
|
||||
batch_processed: bool = False,
|
||||
) -> None:
|
||||
"""Persist a managed object row, caching it and upserting it in the DB.
|
||||
|
||||
|
|
@ -328,6 +339,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
row absent from the table is left absent rather than created with the
|
||||
observer as its creator, because created_by and team_id are written from
|
||||
whoever calls the create branch.
|
||||
|
||||
batch_processed is set by callers that have already billed the batch
|
||||
themselves, so CheckBatchCost skips the row instead of billing it twice.
|
||||
It is written only in the upsert create branch.
|
||||
"""
|
||||
verbose_logger.info(f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache")
|
||||
litellm_managed_object = LiteLLM_ManagedObjectTable(
|
||||
|
|
@ -379,6 +394,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
"updated_by": user_api_key_dict.user_id,
|
||||
"status": file_object.status,
|
||||
**attribution_columns,
|
||||
"batch_processed": batch_processed,
|
||||
},
|
||||
"update": update_columns,
|
||||
},
|
||||
|
|
@ -1343,6 +1359,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes
|
||||
) -> LLMResponseTypes:
|
||||
if isinstance(response, LiteLLMBatch):
|
||||
decoded_batch_id: Final = _is_base64_encoded_unified_file_id(response.id)
|
||||
if decoded_batch_id and is_litellm_executed_batch(decoded_batch_id):
|
||||
return response
|
||||
## Check if unified_file_id is in the response
|
||||
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
|
||||
unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id
|
||||
|
|
@ -1794,24 +1813,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# Check if file deletion should be blocked due to batch references
|
||||
await self._check_file_deletion_allowed(file_id)
|
||||
|
||||
# file_id = convert_b64_uid_to_unified_uid(file_id)
|
||||
model_file_id_mapping = await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span)
|
||||
|
||||
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
|
||||
if specific_model_file_id_mapping:
|
||||
# Remove conflicting keys from data to avoid duplicate keyword arguments
|
||||
filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")}
|
||||
for model_id, model_file_id in specific_model_file_id_mapping.items():
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
delete_data = {
|
||||
**{k: v for k, v in filtered_data.items() if k != "_litellm_internal_model_credentials"},
|
||||
**(
|
||||
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
|
||||
if credentials is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
|
||||
managed_file: Final = await self.get_unified_file_id(file_id, litellm_parent_otel_span)
|
||||
if managed_file is not None and managed_file.storage_backend and managed_file.storage_url:
|
||||
await self._delete_storage_backend_content(managed_file.storage_backend, managed_file.storage_url)
|
||||
else:
|
||||
await self._delete_provider_files(file_id, litellm_parent_otel_span, llm_router, data)
|
||||
|
||||
await self.delete_unified_file_id(file_id, litellm_parent_otel_span)
|
||||
|
||||
|
|
@ -1820,16 +1826,53 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
prom_logger.record_managed_file_deleted(result="success")
|
||||
return FileDeleted(id=file_id, object="file", deleted=True)
|
||||
|
||||
async def _delete_storage_backend_content(self, storage_backend_name: str, storage_url: str) -> None:
|
||||
try:
|
||||
storage_backend: Final = get_storage_backend(storage_backend_name, prisma_client=self.prisma_client)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=f"Cannot delete the stored file content: {e}") from e
|
||||
await storage_backend.delete_file(storage_url)
|
||||
|
||||
async def _delete_provider_files(
|
||||
self,
|
||||
file_id: str,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
llm_router: Router,
|
||||
data: Mapping[str, object],
|
||||
) -> None:
|
||||
model_file_id_mapping: Final = await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span)
|
||||
specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
|
||||
if not specific_model_file_id_mapping:
|
||||
return
|
||||
filtered_data: Final = {
|
||||
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
|
||||
}
|
||||
for model_id, model_file_id in specific_model_file_id_mapping.items():
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
delete_data = {
|
||||
**filtered_data,
|
||||
**(
|
||||
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
|
||||
if credentials is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
|
||||
|
||||
async def afile_content(
|
||||
self,
|
||||
file_id: str,
|
||||
litellm_parent_otel_span: Optional[Span],
|
||||
llm_router: Router,
|
||||
**data: Dict,
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
) -> HttpxBinaryResponseContent:
|
||||
"""
|
||||
Get the content of a file from first model that has it
|
||||
"""
|
||||
managed_file: Final = await self.get_unified_file_id(file_id, litellm_parent_otel_span)
|
||||
if managed_file is not None and managed_file.storage_backend and managed_file.storage_url:
|
||||
return await self._storage_backend_content(managed_file.storage_backend, managed_file.storage_url)
|
||||
|
||||
model_file_id_mapping = data.pop("model_file_id_mapping", None)
|
||||
model_file_id_mapping = model_file_id_mapping or await self.get_model_file_id_mapping(
|
||||
[file_id], litellm_parent_otel_span
|
||||
|
|
@ -1859,6 +1902,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
else:
|
||||
raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
|
||||
|
||||
async def _storage_backend_content(self, storage_backend_name: str, storage_url: str) -> HttpxBinaryResponseContent:
|
||||
storage_backend: Final = get_storage_backend(storage_backend_name, prisma_client=self.prisma_client)
|
||||
content: Final = await storage_backend.download_file(storage_url)
|
||||
return HttpxBinaryResponseContent(response=httpx.Response(status_code=httpx.codes.OK, content=content))
|
||||
|
||||
async def _convert_storage_files_to_base64(
|
||||
self,
|
||||
messages: List[AllMessageValues],
|
||||
|
|
@ -1889,16 +1937,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
|
||||
# File is stored in a storage backend, download and convert to base64
|
||||
try:
|
||||
from litellm.llms.base_llm.files.storage_backend_factory import (
|
||||
get_storage_backend,
|
||||
)
|
||||
|
||||
storage_backend_name = db_file.storage_backend
|
||||
storage_url = db_file.storage_url
|
||||
|
||||
# Get storage backend (uses same env vars as callback)
|
||||
try:
|
||||
storage_backend = get_storage_backend(storage_backend_name)
|
||||
storage_backend = get_storage_backend(storage_backend_name, prisma_client=self.prisma_client)
|
||||
except ValueError as e:
|
||||
verbose_logger.warning(
|
||||
f"Storage backend '{storage_backend_name}' error for file {file_id}: {str(e)}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
ALTER TABLE "LiteLLM_AutoRouterSession"
|
||||
ADD COLUMN IF NOT EXISTS "savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS "savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS "savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS "savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}';
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterBaselineComparison" (
|
||||
"scope" TEXT PRIMARY KEY,
|
||||
"api_key" TEXT NOT NULL,
|
||||
"session_id" TEXT NOT NULL,
|
||||
"router_name" TEXT NOT NULL,
|
||||
"initial_equivalent" BOOLEAN NOT NULL,
|
||||
"revision" BIGINT NOT NULL DEFAULT 0,
|
||||
"published_revision" BIGINT NOT NULL DEFAULT 0,
|
||||
"history" TEXT,
|
||||
"attempted_at" TIMESTAMP(3),
|
||||
"retired" BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_scope"
|
||||
ON "LiteLLM_AutoRouterBaselineComparison" ("api_key", "session_id", "router_name");
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_updated"
|
||||
ON "LiteLLM_AutoRouterBaselineComparison" ("updated_at");
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_dirty"
|
||||
ON "LiteLLM_AutoRouterBaselineComparison" ("attempted_at", "updated_at", "scope")
|
||||
WHERE NOT "retired" AND "revision" <> "published_revision";
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterBaselineObservation" (
|
||||
"request_id" TEXT PRIMARY KEY,
|
||||
"scope" TEXT NOT NULL,
|
||||
"started_at" DOUBLE PRECISION NOT NULL,
|
||||
"revision" BIGINT NOT NULL,
|
||||
"data" TEXT NOT NULL,
|
||||
"publication" TEXT,
|
||||
"conflicted" BOOLEAN NOT NULL DEFAULT FALSE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_event_order"
|
||||
ON "LiteLLM_AutoRouterBaselineObservation" ("scope", "started_at", "request_id");
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_event_revision"
|
||||
ON "LiteLLM_AutoRouterBaselineObservation" ("scope", "revision", "started_at");
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedFileContentTable" (
|
||||
"id" TEXT NOT NULL,
|
||||
"content" BYTEA NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "LiteLLM_ManagedFileContentTable_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
|
@ -1107,6 +1107,12 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
|
|||
@@index([team_id, created_at(sort: Desc)])
|
||||
}
|
||||
|
||||
model LiteLLM_ManagedFileContentTable {
|
||||
id String @id @default(uuid())
|
||||
content Bytes
|
||||
created_at DateTime @default(now())
|
||||
}
|
||||
|
||||
model LiteLLM_ManagedVectorStoreTable {
|
||||
id String @id @default(uuid())
|
||||
unified_resource_id String @unique // The base64 encoded unified vector store ID
|
||||
|
|
@ -1545,6 +1551,36 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterBaselineComparison {
|
||||
scope String @id
|
||||
api_key String
|
||||
session_id String
|
||||
router_name String
|
||||
initial_equivalent Boolean
|
||||
revision BigInt @default(0)
|
||||
published_revision BigInt @default(0)
|
||||
history String?
|
||||
attempted_at DateTime?
|
||||
retired Boolean @default(false)
|
||||
updated_at DateTime @default(now())
|
||||
|
||||
@@index([api_key, session_id, router_name], map: "idx_autorouter_baseline_scope")
|
||||
@@index([updated_at], map: "idx_autorouter_baseline_updated")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterBaselineObservation {
|
||||
request_id String @id
|
||||
scope String
|
||||
started_at Float
|
||||
revision BigInt
|
||||
data String
|
||||
publication String?
|
||||
conflicted Boolean @default(false)
|
||||
|
||||
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")
|
||||
@@index([scope, revision, started_at], map: "idx_autorouter_baseline_event_revision")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterSession {
|
||||
api_key String
|
||||
session_id String
|
||||
|
|
@ -1571,6 +1607,10 @@ model LiteLLM_AutoRouterSession {
|
|||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
savings_estimated_baseline_models Json @default("{}")
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
|
|
|
|||
|
|
@ -1694,6 +1694,7 @@ LOGIN_THROTTLE_NOT_BLOCKED: Final = (0, 0)
|
|||
LITELLM_PROXY_ADMIN_NAME: Final = "default_user_id"
|
||||
LITELLM_PROXY_BUDGET_NAME: Final = "litellm-proxy-budget"
|
||||
GLOBAL_PROXY_SPEND_CACHE_KEY: Final = f"{LITELLM_PROXY_ADMIN_NAME}:spend"
|
||||
LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, int(os.getenv("LITELLM_EXECUTED_BATCH_CONCURRENCY", "4")))
|
||||
|
||||
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
|
||||
LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli"
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ if TYPE_CHECKING:
|
|||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
|
||||
from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
|
||||
try:
|
||||
from litellm_enterprise.enterprise_callbacks.callback_controls import (
|
||||
EnterpriseCallbackControls,
|
||||
|
|
@ -501,6 +502,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_request_debug: bool = False
|
||||
streamed_anthropic_message_id: str | None = None
|
||||
classifier_input: Mapping[str, JsonValue] | None = None
|
||||
baseline_cache_context: "BaselineCacheContext | None" = None
|
||||
baseline_observation: "CapturedBaselineObservation | None" = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -508,7 +511,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
messages,
|
||||
stream,
|
||||
call_type,
|
||||
start_time,
|
||||
start_time: datetime.datetime,
|
||||
litellm_call_id: str,
|
||||
function_id: str,
|
||||
litellm_trace_id: str | None = None,
|
||||
|
|
@ -2181,6 +2184,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
logging_result,
|
||||
start_time,
|
||||
end_time,
|
||||
build_logging_payload: bool = True,
|
||||
):
|
||||
"""Resolve hidden params, compute response cost, and emit the standard logging payload."""
|
||||
hidden_params: Final = getattr(logging_result, "_hidden_params", {})
|
||||
|
|
@ -2205,6 +2209,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
else:
|
||||
self.model_call_details["response_cost"] = self._response_cost_calculator(result=logging_result)
|
||||
|
||||
if not build_logging_payload:
|
||||
return
|
||||
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
logging_result, start_time, end_time
|
||||
)
|
||||
|
|
@ -2215,6 +2222,19 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if standard_logging_payload is not None:
|
||||
emit_standard_logging_payload(standard_logging_payload)
|
||||
|
||||
async def _prepare_baseline_cache_estimate(self, response_obj: object) -> None:
|
||||
if self.baseline_cache_context is None:
|
||||
return
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import finalize_baseline_cache
|
||||
|
||||
await finalize_baseline_cache(self, response_obj)
|
||||
|
||||
async def invalidate_baseline_cache_estimate(self, reason: str, *, completed: bool = False) -> None:
|
||||
"""Invalidate uncertain attempts; retire the reservation at logical completion."""
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import invalidate_baseline_cache
|
||||
|
||||
await invalidate_baseline_cache(self, reason, completed=completed)
|
||||
|
||||
def _build_standard_logging_payload(
|
||||
self, init_response_obj: object, start_time: Any, end_time: Any
|
||||
) -> StandardLoggingPayload | None:
|
||||
|
|
@ -2266,6 +2286,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time=None,
|
||||
cache_hit=None,
|
||||
standard_logging_object: StandardLoggingPayload | None = None,
|
||||
build_logging_payload: bool = True,
|
||||
):
|
||||
try:
|
||||
if start_time is None:
|
||||
|
|
@ -2303,6 +2324,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
logging_result=logging_result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
build_logging_payload=build_logging_payload,
|
||||
)
|
||||
elif standard_logging_object is not None:
|
||||
self.model_call_details["standard_logging_object"] = standard_logging_object
|
||||
|
|
@ -3051,8 +3073,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
result=result,
|
||||
cache_hit=cache_hit,
|
||||
standard_logging_object=kwargs.get("standard_logging_object", None),
|
||||
build_logging_payload=self.baseline_cache_context is None,
|
||||
)
|
||||
|
||||
if self.stream is not True and self.baseline_cache_context is not None:
|
||||
await self._prepare_baseline_cache_estimate(result)
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
)
|
||||
if (prepared_payload := self.model_call_details.get("standard_logging_object")) is not None:
|
||||
emit_standard_logging_payload(prepared_payload)
|
||||
|
||||
## BUILD COMPLETE STREAMED RESPONSE
|
||||
if "async_complete_streaming_response" in self.model_call_details:
|
||||
return # break out of this.
|
||||
|
|
@ -3097,6 +3128,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
|
||||
await self._prepare_baseline_cache_estimate(complete_streaming_response)
|
||||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
try:
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
|
|
@ -3125,6 +3158,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# Only build standard_logging_object if not already built by
|
||||
# _success_handler_helper_fn
|
||||
if self.model_call_details.get("standard_logging_object") is None:
|
||||
await self._prepare_baseline_cache_estimate(result)
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
|
|
@ -3631,6 +3665,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"""
|
||||
Implementing async callbacks, to handle asyncio event loop issues when custom integrations need to use async functions.
|
||||
"""
|
||||
if self.baseline_cache_context is not None:
|
||||
await self.invalidate_baseline_cache_estimate("failed_request")
|
||||
await self.special_failure_handlers(exception=exception)
|
||||
if not self.should_run_logging(event_type="async_failure"): # prevent double logging
|
||||
return
|
||||
|
|
@ -5528,6 +5564,10 @@ class StandardLoggingPayloadSetup:
|
|||
for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
|
||||
clean_metadata[key] = metadata[key]
|
||||
|
||||
recorded_guardrails: Final = metadata.get("applied_guardrails")
|
||||
if applied_guardrails and isinstance(recorded_guardrails, list):
|
||||
clean_metadata["applied_guardrails"] = list(dict.fromkeys([*applied_guardrails, *recorded_guardrails]))
|
||||
|
||||
user_api_key: Final = metadata.get("user_api_key")
|
||||
if user_api_key and isinstance(user_api_key, str) and is_valid_sha256_hash(user_api_key):
|
||||
clean_metadata["user_api_key_hash"] = user_api_key
|
||||
|
|
@ -6149,6 +6189,8 @@ def _autorouter_savings_for_payload(
|
|||
model_id: str | None,
|
||||
usage_object: Mapping[str, object] | None,
|
||||
cost_breakdown: Mapping[str, object] | None,
|
||||
baseline_usage: Usage | None = None,
|
||||
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
|
||||
) -> float | None:
|
||||
"""The auto-router savings figure for the payload, or ``None`` when there is none.
|
||||
|
||||
|
|
@ -6167,6 +6209,8 @@ def _autorouter_savings_for_payload(
|
|||
model_id=model_id,
|
||||
usage_object=usage_object,
|
||||
cost_breakdown=cost_breakdown,
|
||||
baseline_usage=baseline_usage,
|
||||
baseline_provenance=baseline_provenance,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging
|
||||
verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
|
||||
|
|
@ -6343,13 +6387,18 @@ def get_standard_logging_object_payload(
|
|||
model_name = response_model_name
|
||||
|
||||
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
|
||||
autorouter_savings: Final = _autorouter_savings_for_payload(
|
||||
request_metadata=metadata,
|
||||
model=model_name,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_id=_model_id,
|
||||
usage_object=usage_dict,
|
||||
cost_breakdown=request_cost_breakdown,
|
||||
captured_baseline: Final = logging_obj.baseline_observation
|
||||
autorouter_savings: Final = (
|
||||
None
|
||||
if status != "success" or cache_hit or logging_obj.baseline_cache_context is not None
|
||||
else _autorouter_savings_for_payload(
|
||||
request_metadata=metadata,
|
||||
model=model_name,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_id=_model_id,
|
||||
usage_object=usage_dict,
|
||||
cost_breakdown=request_cost_breakdown,
|
||||
)
|
||||
)
|
||||
|
||||
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
|
||||
|
|
@ -6396,6 +6445,26 @@ def get_standard_logging_object_payload(
|
|||
response_cost=response_cost,
|
||||
cost_breakdown=request_cost_breakdown,
|
||||
autorouter_savings=autorouter_savings,
|
||||
autorouter_savings_estimate=(
|
||||
{
|
||||
"version": 3,
|
||||
"status": "unknown",
|
||||
"reason": "pending_projection",
|
||||
} # mutable-ok: spend-log JSON serialization requires plain mappings
|
||||
if captured_baseline is not None
|
||||
else (
|
||||
{ # mutable-ok: spend-log JSON serialization requires plain mappings
|
||||
"version": 1,
|
||||
"status": "estimated" if autorouter_savings is not None else "unknown",
|
||||
"reason": "uncached_usage" if autorouter_savings is not None else "baseline_unavailable",
|
||||
}
|
||||
if metadata.get("routing_decision")
|
||||
else None
|
||||
)
|
||||
),
|
||||
autorouter_baseline_observation=(
|
||||
captured_baseline.model_dump_json() if captured_baseline is not None else None
|
||||
),
|
||||
total_tokens=usage_dict.get("total_tokens", 0),
|
||||
prompt_tokens=usage_dict.get("prompt_tokens", 0),
|
||||
completion_tokens=usage_dict.get("completion_tokens", 0),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ Calling + translation logic for anthropic's `/v1/messages` endpoint
|
|||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -31,7 +32,6 @@ from litellm.types.llms.anthropic import (
|
|||
ContentBlockStop,
|
||||
MessageBlockDelta,
|
||||
MessageStartBlock,
|
||||
UsageDelta,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
|
|
@ -557,6 +557,7 @@ class ModelResponseIterator:
|
|||
self.tool_index = -1
|
||||
self.json_mode = json_mode
|
||||
self.speed = speed
|
||||
self._cumulative_usage: Mapping[str, object] = MappingProxyType({})
|
||||
# rewritten-name -> caller's original. Built per-request from the
|
||||
# forward map in AnthropicConfig._build_request_tool_name_maps; only
|
||||
# contains entries we actually rewrote, so a tool legitimately named
|
||||
|
|
@ -631,10 +632,12 @@ class ModelResponseIterator:
|
|||
return True
|
||||
return False
|
||||
|
||||
def _handle_usage(self, anthropic_usage_chunk: dict | UsageDelta) -> Usage:
|
||||
def _handle_usage(self, anthropic_usage_chunk: Mapping[str, object]) -> Usage:
|
||||
# message_delta usage is cumulative but may omit fields reported at message_start.
|
||||
self._cumulative_usage = MappingProxyType({**self._cumulative_usage, **anthropic_usage_chunk})
|
||||
reasoning_content: Final = "".join(self.reasoning_content_chunks) if self.reasoning_content_chunks else None
|
||||
usage: Final = AnthropicConfig().calculate_usage(
|
||||
usage_object=cast(dict, anthropic_usage_chunk),
|
||||
usage_object=self._cumulative_usage,
|
||||
reasoning_content=reasoning_content,
|
||||
speed=self.speed,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@ Anthropic CountTokens API handler.
|
|||
Uses httpx for HTTP requests instead of the Anthropic SDK.
|
||||
"""
|
||||
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -16,6 +18,8 @@ from litellm.llms.anthropic.count_tokens.transformation import (
|
|||
)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
||||
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
||||
"""
|
||||
|
|
@ -27,13 +31,14 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
async def handle_count_tokens_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict[str, Any]],
|
||||
messages: list[dict[str, JsonValue]],
|
||||
api_key: str,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
system: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
Handle a CountTokens request using httpx.
|
||||
|
||||
|
|
@ -52,7 +57,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
"""
|
||||
try:
|
||||
# Validate the request
|
||||
self.validate_request(model, messages)
|
||||
self.validate_request(model, messages, system=system, tools=tools)
|
||||
|
||||
verbose_logger.debug("Processing Anthropic CountTokens request for model: %s", model)
|
||||
|
||||
|
|
@ -62,6 +67,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
verbose_logger.debug("Transformed request: %s", request_body)
|
||||
|
|
@ -97,7 +103,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
message=error_text,
|
||||
)
|
||||
|
||||
anthropic_response: Final = response.json()
|
||||
anthropic_response: Final = _COUNT_RESPONSE.validate_json(response.content)
|
||||
|
||||
verbose_logger.debug("Anthropic response: %s", anthropic_response)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,17 @@ Anthropic CountTokens API transformation logic.
|
|||
This module handles the transformation of requests to Anthropic's CountTokens API format.
|
||||
"""
|
||||
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
|
||||
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
|
||||
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
|
||||
|
||||
|
||||
class AnthropicCountTokensConfig:
|
||||
"""
|
||||
|
|
@ -31,27 +38,31 @@ class AnthropicCountTokensConfig:
|
|||
def transform_request_to_count_tokens(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
system: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
messages: list[dict[str, JsonValue]],
|
||||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
|
||||
"""
|
||||
Transform request to Anthropic CountTokens format.
|
||||
|
||||
Includes optional system and tools fields for accurate token counting.
|
||||
"""
|
||||
request: Final[dict[str, Any]] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
if system is not None:
|
||||
request["system"] = system
|
||||
|
||||
if tools is not None:
|
||||
request["tools"] = tools
|
||||
|
||||
return request
|
||||
options: Final[Mapping[str, JsonValue]] = optional_params or MappingProxyType({})
|
||||
return _COUNT_REQUEST.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
|
||||
),
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in options.items() if key in COUNT_TOKEN_OPTION_NAMES}
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def get_required_headers(self, api_key: str) -> dict[str, str]:
|
||||
"""
|
||||
|
|
@ -76,7 +87,14 @@ class AnthropicCountTokensConfig:
|
|||
headers, _ = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
|
||||
return headers
|
||||
|
||||
def validate_request(self, model: str, messages: list[dict[str, Any]]) -> None:
|
||||
def validate_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
*,
|
||||
system: JsonValue = None,
|
||||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Validate the incoming count tokens request.
|
||||
|
||||
|
|
@ -90,7 +108,7 @@ class AnthropicCountTokensConfig:
|
|||
if not model:
|
||||
raise ValueError("model parameter is required")
|
||||
|
||||
if not messages:
|
||||
if not messages and not system and not tools:
|
||||
raise ValueError("messages parameter is required")
|
||||
|
||||
if not isinstance(messages, list):
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import accumulate, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias
|
||||
|
||||
|
|
@ -14,9 +15,14 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, StrictInt, TypeAda
|
|||
import litellm
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, is_anthropic_oauth_key
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION
|
||||
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import supports_thinking_cache_preservation
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_HEADERS: Final = TypeAdapter(dict[str, str])
|
||||
|
|
@ -100,10 +106,7 @@ _Block: TypeAlias = Annotated[_Text | _ToolUse | _ToolResult, Field(discriminato
|
|||
|
||||
class _Message(_StrictModel):
|
||||
role: Literal["user", "assistant"]
|
||||
content: str | Annotated[tuple[_Block, ...], Field(strict=False)]
|
||||
|
||||
def blocks(self) -> tuple[_Text | _ToolUse | _ToolResult, ...]:
|
||||
return (_Text(type="text", text=self.content),) if isinstance(self.content, str) else tuple(self.content)
|
||||
content: Annotated[str, Field(min_length=1, pattern=r"\S")] | Annotated[tuple[_Block, ...], Field(strict=False)]
|
||||
|
||||
|
||||
class _Tool(_StrictModel):
|
||||
|
|
@ -113,10 +116,7 @@ class _Tool(_StrictModel):
|
|||
type: Literal["custom"] | None = None
|
||||
|
||||
|
||||
class _Request(_StrictModel):
|
||||
messages: tuple[_Message, ...] = Field(min_length=1, strict=False)
|
||||
system: str | Annotated[tuple[_ResultText, ...], Field(strict=False)] | None = None
|
||||
tools: Annotated[tuple[_Tool, ...], Field(strict=False)] | None = None
|
||||
class _RequestOptions(_StrictModel):
|
||||
model: str | None = None
|
||||
max_tokens: int | None = None
|
||||
stream: bool | None = None
|
||||
|
|
@ -127,6 +127,289 @@ class _Request(_StrictModel):
|
|||
metadata: Mapping[str, JsonValue] | None = None
|
||||
|
||||
|
||||
class _Request(_RequestOptions):
|
||||
messages: tuple[_Message, ...] = Field(min_length=1, strict=False)
|
||||
system: str | Annotated[tuple[_ResultText, ...], Field(strict=False)] | None = None
|
||||
tools: Annotated[tuple[_Tool, ...], Field(strict=False)] | None = None
|
||||
|
||||
|
||||
class _Thinking(_StrictModel):
|
||||
type: Literal["thinking"]
|
||||
thinking: str
|
||||
signature: str = Field(min_length=1)
|
||||
|
||||
|
||||
_PlanBlock: TypeAlias = Annotated[_Text | _ToolUse | _ToolResult | _Thinking, Field(discriminator="type")]
|
||||
|
||||
|
||||
class _PlanMessage(_StrictModel):
|
||||
role: Literal["user", "assistant", "system"]
|
||||
content: str | Annotated[tuple[_PlanBlock, ...], Field(strict=False)]
|
||||
|
||||
|
||||
class _PlanTool(_Tool):
|
||||
cache_control: _CacheControl | None = None
|
||||
|
||||
|
||||
class _PlanRequest(_RequestOptions):
|
||||
messages: tuple[_PlanMessage, ...] = Field(min_length=1, strict=False)
|
||||
system: str | Annotated[tuple[_Text, ...], Field(strict=False)] | None = None
|
||||
tools: Annotated[tuple[_PlanTool, ...], Field(strict=False)] | None = None
|
||||
cache_control: _CacheControl | None = None
|
||||
thinking: Mapping[str, JsonValue] | None = None
|
||||
tool_choice: Mapping[str, JsonValue] | None = None
|
||||
output_config: Mapping[str, JsonValue] | None = None
|
||||
speed: Literal["fast", "standard"] | None = None
|
||||
service_tier: Literal["auto", "standard_only"] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CacheBoundary:
|
||||
fingerprint: str
|
||||
prefix_body: Mapping[str, JsonValue] = field(repr=False)
|
||||
ttl_seconds: int
|
||||
lookback_fingerprints: tuple[str, ...]
|
||||
content_fingerprint: str = ""
|
||||
lookback_content_fingerprints: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PromptCachePlan:
|
||||
full_body: Mapping[str, JsonValue] = field(repr=False)
|
||||
breakpoints: tuple[CacheBoundary, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnsupportedCachePlan:
|
||||
reason: Literal[
|
||||
"unsupported_prompt_shape",
|
||||
"conflicting_cache_ttl",
|
||||
"too_many_cache_breakpoints",
|
||||
"invalid_cache_ttl_order",
|
||||
"unsupported_thinking_cache_semantics",
|
||||
"token_count_unavailable",
|
||||
"inconsistent_prefix_token_count",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CountedBreakpoint:
|
||||
fingerprint: str
|
||||
ttl_seconds: int
|
||||
prefix_tokens: int
|
||||
lookback_fingerprints: tuple[str, ...]
|
||||
content_fingerprint: str = ""
|
||||
lookback_content_fingerprints: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CountedPromptCachePlan:
|
||||
total_tokens: int
|
||||
breakpoints: tuple[CountedBreakpoint, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Position:
|
||||
section: Literal["tools", "system", "messages"]
|
||||
message_index: int
|
||||
role: str
|
||||
block: Mapping[str, JsonValue]
|
||||
marker: _CacheControl | None
|
||||
|
||||
|
||||
def _content_blocks(content: JsonValue) -> tuple[Mapping[str, JsonValue], ...]:
|
||||
if isinstance(content, str):
|
||||
return (MappingProxyType({"type": "text", "text": content}),)
|
||||
return tuple(_JSON_OBJECT.validate_python(block) for block in content) if isinstance(content, list) else ()
|
||||
|
||||
|
||||
def _position(
|
||||
section: Literal["tools", "system", "messages"],
|
||||
message_index: int,
|
||||
role: str,
|
||||
block: Mapping[str, JsonValue],
|
||||
) -> _Position:
|
||||
control: Final = block.get("cache_control")
|
||||
return _Position(
|
||||
section,
|
||||
message_index,
|
||||
role,
|
||||
MappingProxyType({key: value for key, value in block.items() if key != "cache_control"}),
|
||||
_CacheControl.model_validate(control) if control is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _positions(body: Mapping[str, JsonValue]) -> tuple[_Position, ...]:
|
||||
tools: Final = body.get("tools")
|
||||
messages: Final = body.get("messages")
|
||||
return (
|
||||
*tuple(
|
||||
_position("tools", -1, "", _JSON_OBJECT.validate_python(tool))
|
||||
for tool in (tools if isinstance(tools, list) else ())
|
||||
),
|
||||
*tuple(_position("system", -1, "", block) for block in _content_blocks(body.get("system"))),
|
||||
*tuple(
|
||||
_position("messages", message_index, str(message.get("role")), block)
|
||||
for message_index, raw_message in enumerate(messages if isinstance(messages, list) else ())
|
||||
for message in (_JSON_OBJECT.validate_python(raw_message),)
|
||||
for block in _content_blocks(message.get("content"))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _prefix_body(
|
||||
body: Mapping[str, JsonValue],
|
||||
positions: tuple[_Position, ...],
|
||||
last_index: int,
|
||||
) -> Mapping[str, JsonValue]:
|
||||
prefix: Final = positions[: last_index + 1]
|
||||
sections: Final = MappingProxyType(
|
||||
{
|
||||
section: _count_objects(tuple(position.block for position in prefix if position.section == section))
|
||||
for section in ("tools", "system")
|
||||
if any(position.section == section for position in prefix)
|
||||
}
|
||||
)
|
||||
messages: Final = tuple(
|
||||
MappingProxyType(
|
||||
_JSON_OBJECT.validate_python(
|
||||
MappingProxyType(
|
||||
{"role": group[0].role, "content": _count_objects(tuple(position.block for position in group))}
|
||||
)
|
||||
)
|
||||
)
|
||||
for _, values in groupby(
|
||||
(position for position in prefix if position.section == "messages"),
|
||||
key=lambda position: position.message_index,
|
||||
)
|
||||
for group in (tuple(values),)
|
||||
)
|
||||
return MappingProxyType(
|
||||
_JSON_OBJECT.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
**MappingProxyType({key: body[key] for key in COUNT_TOKEN_OPTION_NAMES if key in body}),
|
||||
**sections,
|
||||
"messages": _count_objects(messages),
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _position_group(position: _Position, index: int) -> tuple[str, int, str | int]:
|
||||
block_type: Final = position.block.get("type")
|
||||
return (
|
||||
position.section,
|
||||
position.message_index,
|
||||
block_type if isinstance(block_type, str) and block_type in ("tool_use", "tool_result") else index,
|
||||
)
|
||||
|
||||
|
||||
def _chain_digest(previous: str, current: str) -> str:
|
||||
return _digest((previous, current))
|
||||
|
||||
|
||||
def _cacheable_position(position: _Position) -> bool:
|
||||
block_type: Final = position.block.get("type")
|
||||
if block_type == "thinking":
|
||||
return False
|
||||
text: Final = position.block.get("text")
|
||||
return block_type != "text" or (isinstance(text, str) and bool(text.strip()))
|
||||
|
||||
|
||||
def _entry_fingerprint(fingerprint: str, ttl_seconds: int) -> str:
|
||||
return _digest(("native-cache-prefix-v2", fingerprint, ttl_seconds))
|
||||
|
||||
|
||||
def parse_cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan | UnsupportedCachePlan:
|
||||
try:
|
||||
request: Final = _PlanRequest.model_validate(body)
|
||||
positions: Final = _positions(body)
|
||||
except ValidationError:
|
||||
return UnsupportedCachePlan("unsupported_prompt_shape")
|
||||
explicit: Final = tuple(
|
||||
(index, position.marker) for index, position in enumerate(positions) if position.marker is not None
|
||||
)
|
||||
automatic_index: Final = next(
|
||||
(index for index in reversed(range(len(positions))) if _cacheable_position(positions[index])), None
|
||||
)
|
||||
automatic_existing: Final = next((marker for index, marker in explicit if index == automatic_index), None)
|
||||
if (
|
||||
request.cache_control is not None
|
||||
and automatic_existing is not None
|
||||
and automatic_existing != request.cache_control
|
||||
):
|
||||
return UnsupportedCachePlan("conflicting_cache_ttl")
|
||||
automatic: Final = (
|
||||
((automatic_index, request.cache_control),)
|
||||
if (request.cache_control is not None and automatic_index is not None and automatic_existing is None)
|
||||
else ()
|
||||
)
|
||||
markers: Final = tuple(sorted((*explicit, *automatic), key=lambda value: value[0]))
|
||||
if len(markers) > 4:
|
||||
return UnsupportedCachePlan("too_many_cache_breakpoints")
|
||||
ttls: Final = tuple(3600 if marker.ttl == "1h" else 300 for _, marker in markers)
|
||||
if any(first < second for first, second in zip(ttls, ttls[1:])):
|
||||
return UnsupportedCachePlan("invalid_cache_ttl_order")
|
||||
settings: Final = MappingProxyType(
|
||||
{
|
||||
key: body[key]
|
||||
for key in ("thinking", "output_config", "speed")
|
||||
if key in body and not (key == "speed" and body[key] == "standard")
|
||||
}
|
||||
)
|
||||
hashes: Final = tuple(
|
||||
accumulate(
|
||||
(
|
||||
_digest(
|
||||
(
|
||||
position.section,
|
||||
position.message_index,
|
||||
position.role,
|
||||
position.block,
|
||||
body.get("tool_choice") if position.section == "messages" else None,
|
||||
)
|
||||
)
|
||||
for position in positions
|
||||
),
|
||||
_chain_digest,
|
||||
initial=_digest(settings),
|
||||
)
|
||||
)[1:]
|
||||
groups: Final = tuple(
|
||||
tuple(index for index, _ in values)
|
||||
for _, values in groupby(
|
||||
enumerate(positions),
|
||||
key=lambda item: _position_group(item[1], item[0]),
|
||||
)
|
||||
)
|
||||
return PromptCachePlan(
|
||||
full_body=MappingProxyType(dict(body)),
|
||||
breakpoints=tuple(
|
||||
CacheBoundary(
|
||||
fingerprint=_entry_fingerprint(hashes[index], ttl),
|
||||
prefix_body=_prefix_body(body, positions, index),
|
||||
ttl_seconds=ttl,
|
||||
lookback_fingerprints=tuple(
|
||||
_entry_fingerprint(hashes[earlier], ttl)
|
||||
for group in reversed(tuple(group for group in groups if group[0] <= index)[-20:])
|
||||
for earlier in reversed(group)
|
||||
if earlier <= index
|
||||
),
|
||||
content_fingerprint=hashes[index],
|
||||
lookback_content_fingerprints=tuple(
|
||||
hashes[earlier]
|
||||
for group in reversed(tuple(group for group in groups if group[0] <= index)[-20:])
|
||||
for earlier in reversed(group)
|
||||
if earlier <= index
|
||||
),
|
||||
)
|
||||
for (index, _), ttl in zip(markers, ttls)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PromptPrefix:
|
||||
prefix_body: Mapping[str, JsonValue]
|
||||
|
|
@ -137,68 +420,28 @@ class PromptPrefix:
|
|||
|
||||
def _digest(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
json.dumps(value, default=_json_object, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _next_digest(previous: str, boundary: tuple[int, str, Mapping[str, JsonValue]]) -> str:
|
||||
return _digest((previous, boundary))
|
||||
def _json_object(value: object) -> dict[str, JsonValue]: # mutable-ok: JSON serialization requires a dictionary
|
||||
return _JSON_OBJECT.validate_python(value)
|
||||
|
||||
|
||||
def parse_prompt(body: Mapping[str, JsonValue]) -> PromptPrefix | None:
|
||||
try:
|
||||
request: Final = _Request.model_validate(body)
|
||||
blocks: Final = tuple(message.blocks() for message in request.messages)
|
||||
_Request.model_validate(body)
|
||||
except ValidationError:
|
||||
return None
|
||||
markers: Final = tuple(
|
||||
(message_index, block_index, block.cache_control)
|
||||
for message_index, message_blocks in enumerate(blocks)
|
||||
for block_index, block in enumerate(message_blocks)
|
||||
if block.cache_control is not None
|
||||
)
|
||||
if len(markers) != 1:
|
||||
plan: Final = parse_cache_plan(body)
|
||||
if isinstance(plan, UnsupportedCachePlan) or len(plan.breakpoints) != 1:
|
||||
return None
|
||||
message_end, block_end, marker = markers[0]
|
||||
normalized: Final = _JSON_OBJECT.validate_python(request.model_dump(mode="json", exclude_none=True))
|
||||
context: Final = MappingProxyType({key: normalized[key] for key in ("system", "tools") if key in normalized})
|
||||
boundaries: Final = tuple(
|
||||
(
|
||||
message_index,
|
||||
request.messages[message_index].role,
|
||||
_JSON_OBJECT.validate_python(
|
||||
block.model_dump(mode="json", exclude=MappingProxyType({"cache_control": True}), exclude_none=True)
|
||||
),
|
||||
)
|
||||
for message_index, message_blocks in enumerate(blocks[: message_end + 1])
|
||||
for block_index, block in enumerate(message_blocks)
|
||||
if message_index < message_end or block_index <= block_end
|
||||
)
|
||||
hashes: Final = tuple(
|
||||
accumulate(boundaries, _next_digest, initial=_digest((_JSON_OBJECT.validate_python(context), marker.ttl)))
|
||||
)[1:]
|
||||
prefix_messages: Final = tuple(
|
||||
_Message(
|
||||
role=request.messages[message_index].role,
|
||||
content=tuple(
|
||||
block
|
||||
for block_index, block in enumerate(message_blocks)
|
||||
if message_index < message_end or block_index <= block_end
|
||||
),
|
||||
)
|
||||
for message_index, message_blocks in enumerate(blocks[: message_end + 1])
|
||||
)
|
||||
prefix: Final = plan.breakpoints[0]
|
||||
return PromptPrefix(
|
||||
prefix_body=MappingProxyType(
|
||||
_JSON_OBJECT.validate_python(
|
||||
_Request(messages=prefix_messages, system=request.system, tools=request.tools).model_dump(
|
||||
mode="json", exclude_none=True
|
||||
)
|
||||
)
|
||||
),
|
||||
fingerprint=hashes[-1],
|
||||
fingerprints=tuple(reversed(hashes[-20:])),
|
||||
ttl_seconds=3600 if marker.ttl == "1h" else 300,
|
||||
prefix_body=prefix.prefix_body,
|
||||
fingerprint=prefix.fingerprint,
|
||||
fingerprints=prefix.lookback_fingerprints,
|
||||
ttl_seconds=prefix.ttl_seconds,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -246,6 +489,9 @@ class _CountBody(BaseModel):
|
|||
messages: Sequence[Mapping[str, JsonValue]]
|
||||
tools: Sequence[Mapping[str, JsonValue]] | None = None
|
||||
system: str | Sequence[Mapping[str, JsonValue]] | None = None
|
||||
thinking: Mapping[str, JsonValue] | None = None
|
||||
tool_choice: Mapping[str, JsonValue] | None = None
|
||||
output_config: Mapping[str, JsonValue] | None = None
|
||||
|
||||
|
||||
class _CountResult(BaseModel):
|
||||
|
|
@ -262,16 +508,36 @@ def _count_objects(
|
|||
return [dict(value) for value in values] # mutable-ok: serialize read-only inputs at the provider API boundary
|
||||
|
||||
|
||||
async def count_prompt_tokens(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
|
||||
native: Final = _CountBody.model_validate(body)
|
||||
def _messages_url(model: str, api_key: str, api_base: str | None) -> str:
|
||||
return AnthropicMessagesConfig().get_complete_url( # pyright: ignore[reportUnknownMemberType] # canonical native URL owner takes legacy JSON arguments
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
optional_params=_JSON_OBJECT.validate_python(MappingProxyType({})),
|
||||
litellm_params=_JSON_OBJECT.validate_python(MappingProxyType({})),
|
||||
)
|
||||
|
||||
|
||||
async def count_prompt_tokens(
|
||||
model: str,
|
||||
api_key: str,
|
||||
body: Mapping[str, JsonValue],
|
||||
api_base: str | None = None,
|
||||
) -> int | None:
|
||||
try:
|
||||
native: Final = _CountBody.model_validate(body)
|
||||
count_url: Final = _messages_url(model, api_key, api_base) + "/count_tokens"
|
||||
result: Final = _CountResult.model_validate(
|
||||
await _counter.handle_count_tokens_request(
|
||||
model=model,
|
||||
messages=_count_objects(native.messages),
|
||||
tools=_count_objects(native.tools) if native.tools is not None else None,
|
||||
system=native.system,
|
||||
system=_JSON_OBJECT.validate_python(MappingProxyType({"system": native.system}))["system"],
|
||||
api_key=api_key,
|
||||
api_base=count_url,
|
||||
optional_params=_JSON_OBJECT.validate_python(
|
||||
MappingProxyType({key: body[key] for key in COUNT_TOKEN_OPTION_NAMES if key in body})
|
||||
),
|
||||
timeout=15.0,
|
||||
)
|
||||
)
|
||||
|
|
@ -280,10 +546,55 @@ async def count_prompt_tokens(model: str, api_key: str, body: Mapping[str, JsonV
|
|||
return result.input_tokens
|
||||
|
||||
|
||||
async def count_cache_plan(
|
||||
model: str,
|
||||
api_key: str,
|
||||
plan: PromptCachePlan,
|
||||
token_counter: TokenCounter = count_prompt_tokens,
|
||||
) -> CountedPromptCachePlan | UnsupportedCachePlan:
|
||||
if any(position.block.get("type") == "thinking" for position in _positions(plan.full_body)):
|
||||
if not supports_thinking_cache_preservation(model, "anthropic"):
|
||||
return UnsupportedCachePlan("unsupported_thinking_cache_semantics")
|
||||
total: Final = await token_counter(model, api_key, plan.full_body)
|
||||
if total is None:
|
||||
return UnsupportedCachePlan("token_count_unavailable")
|
||||
counts: Final = tuple(
|
||||
await asyncio.gather(*(token_counter(model, api_key, marker.prefix_body) for marker in plan.breakpoints))
|
||||
)
|
||||
if any(value is None for value in counts):
|
||||
return UnsupportedCachePlan("token_count_unavailable")
|
||||
known: Final = tuple(value for value in counts if value is not None)
|
||||
if any(value < 0 for value in (total, *known)) or any(
|
||||
first > second for first, second in zip(known, (*known[1:], total))
|
||||
):
|
||||
return UnsupportedCachePlan("inconsistent_prefix_token_count")
|
||||
return CountedPromptCachePlan(
|
||||
total,
|
||||
tuple(
|
||||
CountedBreakpoint(
|
||||
marker.fingerprint,
|
||||
marker.ttl_seconds,
|
||||
count,
|
||||
marker.lookback_fingerprints,
|
||||
marker.content_fingerprint,
|
||||
marker.lookback_content_fingerprints,
|
||||
)
|
||||
for marker, count in zip(plan.breakpoints, known)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativePredictionTarget:
|
||||
model: str
|
||||
api_key: str
|
||||
api_key: str = field(repr=False)
|
||||
api_base: str | None = None
|
||||
|
||||
|
||||
def supported_baseline_recipient(target: NativePredictionTarget, wire: httpx.Request) -> bool:
|
||||
return wire.headers.get("x-api-key") == target.api_key and wire.url == httpx.URL(
|
||||
_messages_url(target.model, target.api_key, target.api_base)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -297,11 +608,26 @@ class UnsupportedPredictionTarget:
|
|||
|
||||
|
||||
def resolve_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget | UnsupportedPredictionTarget:
|
||||
return _resolve_prediction_target(params, allow_configured_endpoint=False)
|
||||
|
||||
|
||||
def resolve_baseline_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget | UnsupportedPredictionTarget:
|
||||
return _resolve_prediction_target(params, allow_configured_endpoint=True)
|
||||
|
||||
|
||||
def _resolve_prediction_target(
|
||||
params: LiteLLM_Params,
|
||||
*,
|
||||
allow_configured_endpoint: bool,
|
||||
) -> NativePredictionTarget | UnsupportedPredictionTarget:
|
||||
configured_options: Final = frozenset(params.model_dump(exclude_defaults=True, exclude_none=True))
|
||||
if configured_options - _DEPLOYMENT_OPTIONS:
|
||||
return UnsupportedPredictionTarget("unsupported_deployment_configuration")
|
||||
api_base: Final = AnthropicModelInfo.get_api_base(params.api_base)
|
||||
if api_base not in ("https://api.anthropic.com", "https://api.anthropic.com/v1/messages"):
|
||||
if not allow_configured_endpoint and api_base not in (
|
||||
"https://api.anthropic.com",
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
):
|
||||
return UnsupportedPredictionTarget("unsupported_provider_endpoint")
|
||||
try:
|
||||
model, provider, _, _ = litellm.get_llm_provider(
|
||||
|
|
@ -314,7 +640,7 @@ def resolve_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget
|
|||
api_key: Final = AnthropicModelInfo.get_api_key(params.api_key)
|
||||
if api_key is None or not _supported_provider_key(api_key):
|
||||
return UnsupportedPredictionTarget("unsupported_provider_credentials")
|
||||
return NativePredictionTarget(model=model, api_key=api_key)
|
||||
return NativePredictionTarget(model=model, api_key=api_key, api_base=api_base)
|
||||
|
||||
|
||||
def _supported_provider_key(api_key: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -280,10 +280,17 @@ class AzureOpenAIConfig(BaseConfig):
|
|||
ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
|
||||
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
|
||||
azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
|
||||
request_params: Final = MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in optional_params.items()
|
||||
if key != "tool_choice" or optional_params.get("tools") or optional_params.get("functions")
|
||||
}
|
||||
)
|
||||
return {
|
||||
"model": model,
|
||||
"messages": azure_messages,
|
||||
**optional_params,
|
||||
**request_params,
|
||||
**sanitized_tools_update(optional_params),
|
||||
}
|
||||
|
||||
|
|
|
|||
40
litellm/llms/base_llm/files/litellm_db_storage_backend.py
Normal file
40
litellm/llms/base_llm/files/litellm_db_storage_backend.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend
|
||||
from litellm.repositories.managed_file_content_repository import ManagedFileContentRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
LITELLM_DB_STORAGE_BACKEND_NAME: Final = "litellm_db"
|
||||
LITELLM_DB_STORAGE_URL_PREFIX: Final = f"{LITELLM_DB_STORAGE_BACKEND_NAME}://"
|
||||
|
||||
|
||||
def storage_url_to_row_id(storage_url: str) -> str:
|
||||
if not storage_url.startswith(LITELLM_DB_STORAGE_URL_PREFIX):
|
||||
raise ValueError(f"Not a {LITELLM_DB_STORAGE_BACKEND_NAME} storage url: {storage_url}")
|
||||
return storage_url.removeprefix(LITELLM_DB_STORAGE_URL_PREFIX)
|
||||
|
||||
|
||||
class LiteLLMDbStorageBackend(BaseFileStorageBackend):
|
||||
def __init__(self, prisma_client: "PrismaClient") -> None:
|
||||
self._contents = ManagedFileContentRepository(prisma_client)
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
file_content: bytes,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
path_prefix: str | None = None,
|
||||
file_naming_strategy: str = "uuid",
|
||||
) -> str:
|
||||
return f"{LITELLM_DB_STORAGE_URL_PREFIX}{await self._contents.store(file_content)}"
|
||||
|
||||
async def download_file(self, storage_url: str) -> bytes:
|
||||
content: Final = await self._contents.load(storage_url_to_row_id(storage_url))
|
||||
if content is None:
|
||||
raise ValueError(f"No stored file content for {storage_url}")
|
||||
return content
|
||||
|
||||
async def delete_file(self, storage_url: str) -> None:
|
||||
await self._contents.delete(storage_url_to_row_id(storage_url))
|
||||
|
|
@ -6,32 +6,46 @@ based on the backend type. Backends use the same configuration as their correspo
|
|||
callbacks (e.g., azure_storage uses the same env vars as AzureBlobStorageLogger).
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
from .azure_blob_storage_backend import AzureBlobStorageBackend
|
||||
from .litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME, LiteLLMDbStorageBackend
|
||||
from .storage_backend import BaseFileStorageBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
def get_storage_backend(backend_type: str) -> BaseFileStorageBackend:
|
||||
|
||||
def get_storage_backend(backend_type: str, prisma_client: "PrismaClient | None" = None) -> BaseFileStorageBackend:
|
||||
"""
|
||||
Factory function to create a storage backend instance.
|
||||
|
||||
Backends are configured using the same environment variables as their
|
||||
corresponding callbacks. For example, "azure_storage" uses the same
|
||||
env vars as AzureBlobStorageLogger.
|
||||
env vars as AzureBlobStorageLogger. "litellm_db" stores file bytes in the
|
||||
proxy's own database and needs the connected Prisma client.
|
||||
|
||||
Args:
|
||||
backend_type: Backend type identifier (e.g., "azure_storage")
|
||||
backend_type: Backend type identifier (e.g., "azure_storage", "litellm_db")
|
||||
prisma_client: The proxy's database client, required by "litellm_db"
|
||||
|
||||
Returns:
|
||||
BaseFileStorageBackend: Instance of the appropriate storage backend
|
||||
|
||||
Raises:
|
||||
ValueError: If backend_type is not supported
|
||||
ValueError: If backend_type is not supported, or "litellm_db" is asked for without a database
|
||||
"""
|
||||
verbose_logger.debug("Creating storage backend: type=%s", backend_type)
|
||||
|
||||
if backend_type == "azure_storage":
|
||||
return AzureBlobStorageBackend()
|
||||
else:
|
||||
raise ValueError(f"Unsupported storage backend type: {backend_type}. Supported types: azure_storage")
|
||||
if backend_type == LITELLM_DB_STORAGE_BACKEND_NAME:
|
||||
if prisma_client is None:
|
||||
raise ValueError(f"Storage backend {LITELLM_DB_STORAGE_BACKEND_NAME} requires a database-connected proxy")
|
||||
return LiteLLMDbStorageBackend(prisma_client)
|
||||
raise ValueError(
|
||||
f"Unsupported storage backend type: {backend_type}. "
|
||||
f"Supported types: azure_storage, {LITELLM_DB_STORAGE_BACKEND_NAME}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2163,6 +2163,8 @@ class BaseLLMHTTPHandler:
|
|||
e=e, litellm_params=litellm_params_dict
|
||||
)
|
||||
if should_retry and not hit_max_attempt:
|
||||
if logging_obj.baseline_cache_context is not None:
|
||||
await logging_obj.invalidate_baseline_cache_estimate("retried_request")
|
||||
verbose_logger.debug(
|
||||
"Anthropic /v1/messages: invalid thinking signature; "
|
||||
"stripping thinking blocks and retrying (attempt %s/%s).",
|
||||
|
|
|
|||
|
|
@ -14510,6 +14510,7 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
|
|
@ -14547,6 +14548,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14698,6 +14700,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14727,6 +14730,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14759,6 +14763,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14796,6 +14801,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14831,6 +14837,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14869,6 +14876,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14986,6 +14994,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -15027,6 +15036,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ maintains per (api_key, session_id, router_name).
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
|
||||
|
|
@ -22,18 +24,25 @@ class LiteLLM_AutoRouterSession(LiteLLMPydanticObjectBase):
|
|||
turns: int
|
||||
spend: float
|
||||
saved_spend: float
|
||||
savings_estimated_turns: int = 0
|
||||
savings_estimated_actual_spend: float = 0.0
|
||||
savings_estimated_saved_spend: float = 0.0
|
||||
savings_estimated_baseline_models: Mapping[str, int] = Field(default_factory=dict)
|
||||
classifier_cost: float
|
||||
tier_turns: Mapping[str, int]
|
||||
baseline_models: Mapping[str, int]
|
||||
|
||||
@property
|
||||
def baseline_model(self) -> str | None:
|
||||
"""The baseline most of this session's turns were priced against, or None when no turn recorded one.
|
||||
"""The baseline most covered turns were priced against, or None when none were estimated.
|
||||
|
||||
A router reconfigured mid-session leaves turns priced against two baselines; the row keeps both
|
||||
counts, and the label is the one that priced the most money-carrying turns rather than whatever the
|
||||
router is configured with now.
|
||||
"""
|
||||
if not self.baseline_models:
|
||||
if not self.savings_estimated_baseline_models:
|
||||
return None
|
||||
return max(self.baseline_models, key=lambda model: (self.baseline_models[model], model))
|
||||
return max(
|
||||
self.savings_estimated_baseline_models,
|
||||
key=lambda model: (self.savings_estimated_baseline_models[model], model),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from pydantic import (
|
|||
ConfigDict,
|
||||
Field,
|
||||
Json,
|
||||
JsonValue,
|
||||
PositiveInt,
|
||||
field_validator,
|
||||
model_validator,
|
||||
|
|
@ -123,6 +124,7 @@ class SupportedDBObjectType(str, enum.Enum):
|
|||
MODEL_COST_MAP = "model_cost_map"
|
||||
TOOLS = "tools"
|
||||
CONFIG_OVERRIDES = "config_overrides"
|
||||
WEBSEARCH_INTERCEPTION_SETTINGS = "websearch_interception_settings"
|
||||
|
||||
def __str__(self):
|
||||
return str(self.value)
|
||||
|
|
@ -3940,6 +3942,7 @@ class SpendLogsRouterMetadata(TypedDict):
|
|||
|
||||
|
||||
class SpendLogsMetadata(TypedDict):
|
||||
autorouter_baseline_observation: ReadOnly[str | None]
|
||||
"""
|
||||
Specific metadata k,v pairs logged to spendlogs for easier cost tracking
|
||||
"""
|
||||
|
|
@ -3980,7 +3983,8 @@ class SpendLogsMetadata(TypedDict):
|
|||
original_model_group: ReadOnly[str | None] # Model group requested before any fallbacks
|
||||
cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
|
||||
compression_savings: CompressionSavingsMetadata | None
|
||||
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
|
||||
autorouter_savings: ReadOnly[float | None]
|
||||
autorouter_savings_estimate: ReadOnly[Mapping[str, JsonValue] | None]
|
||||
litellm_gateway_injected_cache: ReadOnly[str | None]
|
||||
router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model
|
||||
azure_spillover: ReadOnly[AzureSpillover | None] # None = Azure did not report spillover
|
||||
|
|
|
|||
|
|
@ -4012,6 +4012,64 @@ async def get_org_object(
|
|||
return _org_obj
|
||||
|
||||
|
||||
def _last_known_org_cache_key(org_id: str) -> str:
|
||||
return f"org_id:{org_id}:with_budget:last_known"
|
||||
|
||||
|
||||
async def _keep_last_known_org(
|
||||
org: LiteLLM_OrganizationTable, org_id: str, user_api_key_cache: UserApiKeyCache
|
||||
) -> None:
|
||||
cache_key: Final = _last_known_org_cache_key(org_id)
|
||||
held_locally: Final = await user_api_key_cache.async_get_cache(
|
||||
key=cache_key, local_only=True, model_type=LiteLLM_OrganizationTable
|
||||
)
|
||||
if held_locally is not None:
|
||||
return
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=cache_key,
|
||||
value=org,
|
||||
model_type=LiteLLM_OrganizationTable,
|
||||
ttl=get_management_object_ttl(user_api_key_cache),
|
||||
)
|
||||
|
||||
|
||||
async def get_org_object_for_request(
|
||||
org_id: str,
|
||||
prisma_client: PrismaClient,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
parent_otel_span: Span | None,
|
||||
proxy_logging_obj: ProxyLogging | None,
|
||||
) -> LiteLLM_OrganizationTable | None:
|
||||
try:
|
||||
org: Final = await get_org_object(
|
||||
org_id=org_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
include_budget_table=True,
|
||||
)
|
||||
except OrganizationNotFoundError:
|
||||
return None
|
||||
except Exception as e: # noqa: BLE001 # only a DB outage may fail auth here, anything else degrades to no org limits
|
||||
if not PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
|
||||
verbose_proxy_logger.debug("org lookup failed, continuing without org limits", exc_info=True)
|
||||
return None
|
||||
last_known_org: Final = await user_api_key_cache.async_get_cache(
|
||||
key=_last_known_org_cache_key(org_id),
|
||||
model_type=LiteLLM_OrganizationTable,
|
||||
)
|
||||
if last_known_org is not None:
|
||||
return last_known_org
|
||||
if PrismaDBExceptionHandler.should_allow_request_on_db_unavailable():
|
||||
return None
|
||||
raise
|
||||
if org is None:
|
||||
return None
|
||||
await _keep_last_known_org(org, org_id, user_api_key_cache)
|
||||
return org
|
||||
|
||||
|
||||
async def _get_resources_from_access_groups(
|
||||
access_group_ids: Sequence[str],
|
||||
resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
|
||||
|
|
@ -5680,7 +5738,7 @@ async def _project_max_budget_check(
|
|||
if project_object.litellm_budget_table is not None:
|
||||
max_budget = project_object.litellm_budget_table.max_budget
|
||||
|
||||
if max_budget is None or max_budget <= 0 or not math.isfinite(max_budget):
|
||||
if max_budget is None or not math.isfinite(max_budget):
|
||||
return
|
||||
|
||||
from litellm.proxy.proxy_server import get_current_spend
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import (
|
|||
get_jwt_key_mapping_object,
|
||||
get_key_end_user_budget_id,
|
||||
get_object_permission,
|
||||
get_org_object_for_request,
|
||||
get_project_object,
|
||||
get_team_membership,
|
||||
get_team_object,
|
||||
|
|
@ -2611,6 +2612,47 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc
|
|||
return PrismaDBExceptionHandler.should_allow_request_on_db_unavailable()
|
||||
|
||||
|
||||
async def _inherit_org_identity(
|
||||
user_api_key_auth_obj: UserAPIKeyAuth,
|
||||
team_object: LiteLLM_TeamTableCachedObj | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
parent_otel_span: Span | None,
|
||||
proxy_logging_obj: ProxyLogging | None,
|
||||
) -> None:
|
||||
if user_api_key_auth_obj.org_id is None and team_object is not None and team_object.organization_id is not None:
|
||||
user_api_key_auth_obj.org_id = team_object.organization_id
|
||||
already_populated: Final = any(
|
||||
value is not None
|
||||
for value in (
|
||||
user_api_key_auth_obj.organization_alias,
|
||||
user_api_key_auth_obj.organization_max_budget,
|
||||
user_api_key_auth_obj.organization_tpm_limit,
|
||||
user_api_key_auth_obj.organization_rpm_limit,
|
||||
user_api_key_auth_obj.organization_metadata,
|
||||
)
|
||||
)
|
||||
if user_api_key_auth_obj.org_id is None or already_populated or prisma_client is None:
|
||||
return
|
||||
org_object: Final = await get_org_object_for_request(
|
||||
org_id=user_api_key_auth_obj.org_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
if org_object is None:
|
||||
return
|
||||
user_api_key_auth_obj.organization_alias = org_object.organization_alias
|
||||
user_api_key_auth_obj.organization_metadata = org_object.metadata
|
||||
budget: Final = org_object.litellm_budget_table
|
||||
if budget is None:
|
||||
return
|
||||
user_api_key_auth_obj.organization_max_budget = budget.max_budget
|
||||
user_api_key_auth_obj.organization_tpm_limit = budget.tpm_limit
|
||||
user_api_key_auth_obj.organization_rpm_limit = budget.rpm_limit
|
||||
|
||||
|
||||
def is_no_auth_dev_mode(master_key: str | None, general_settings: Mapping[str, object]) -> bool:
|
||||
return master_key is None and not any(
|
||||
general_settings.get(flag, False)
|
||||
|
|
@ -2849,8 +2891,14 @@ async def _run_centralized_common_checks(
|
|||
user_object=user_object,
|
||||
)
|
||||
|
||||
if user_api_key_auth_obj.org_id is None and team_object is not None and team_object.organization_id is not None:
|
||||
user_api_key_auth_obj.org_id = team_object.organization_id
|
||||
await _inherit_org_identity(
|
||||
user_api_key_auth_obj=user_api_key_auth_obj,
|
||||
team_object=team_object,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
# common_checks identifies admin via user_object, not the token
|
||||
# (non_proxy_admin_allowed_routes_check). JWT admin shortcut and
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@
|
|||
import asyncio
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -18,6 +20,15 @@ from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest
|
|||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
|
||||
from litellm.proxy.batches_endpoints.litellm_executed_batches import (
|
||||
LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE,
|
||||
LiteLLMExecutedBatchRunner,
|
||||
ManagedBatchStore,
|
||||
batch_error,
|
||||
executed_batch_runner_lost,
|
||||
litellm_executed_provider_for,
|
||||
resolve_litellm_executed_provider,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
log_llm_api_exception,
|
||||
|
|
@ -47,16 +58,87 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
get_model_id_from_unified_batch_id,
|
||||
get_models_from_unified_file_id,
|
||||
get_original_file_id,
|
||||
is_litellm_executed_batch,
|
||||
prepare_data_with_credentials,
|
||||
update_batch_in_database,
|
||||
validate_managed_id_requirement,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import request_tags_from_metadata
|
||||
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
|
||||
from litellm.proxy.utils import handle_exception_on_proxy, is_known_model
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy, is_known_model
|
||||
from litellm.repositories.managed_batch_repository import ManagedBatchRepository
|
||||
from litellm.repositories.table_repositories import ManagedFileRepository
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.openai import LiteLLMBatchCreateRequest
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma.models import LiteLLM_ManagedObjectTable
|
||||
|
||||
router: Final = APIRouter()
|
||||
_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _request_tags(data: Mapping[str, object]) -> tuple[str, ...] | None:
|
||||
metadata: Final = data.get("litellm_metadata")
|
||||
if metadata is None:
|
||||
return None
|
||||
return request_tags_from_metadata(_METADATA_ADAPTER.validate_python(metadata))
|
||||
|
||||
|
||||
def _litellm_executed_batch_runner(llm_router: Router, proxy_logging_obj: ProxyLogging) -> LiteLLMExecutedBatchRunner:
|
||||
from litellm.proxy.proxy_server import general_settings, prisma_client
|
||||
|
||||
managed_files: Final = proxy_logging_obj.get_proxy_hook("managed_files")
|
||||
if prisma_client is None or not isinstance(managed_files, ManagedBatchStore):
|
||||
raise batch_error(
|
||||
400,
|
||||
"LiteLLM-executed batches need a database: set DATABASE_URL so LiteLLM can keep the batch and its files",
|
||||
)
|
||||
return LiteLLMExecutedBatchRunner(
|
||||
llm_router=llm_router,
|
||||
prisma_client=prisma_client,
|
||||
managed_files=managed_files,
|
||||
batches=ManagedBatchRepository(prisma_client),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings=general_settings,
|
||||
)
|
||||
|
||||
|
||||
async def _batch_from_database(
|
||||
batch_id: str,
|
||||
unified_batch_id: str | Literal[False],
|
||||
executed_batch: bool,
|
||||
managed_files_obj: object,
|
||||
prisma_client: PrismaClient | None,
|
||||
llm_router: Router | None,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> tuple["LiteLLM_ManagedObjectTable | None", LiteLLMBatch | None]:
|
||||
row, batch = await get_batch_from_database(
|
||||
batch_id=batch_id,
|
||||
unified_batch_id=unified_batch_id,
|
||||
managed_files_obj=managed_files_obj,
|
||||
prisma_client=prisma_client,
|
||||
verbose_proxy_logger=verbose_proxy_logger,
|
||||
)
|
||||
updated_at: Final[object] = getattr(row, "updated_at", None)
|
||||
if not executed_batch or batch is None or llm_router is None or not isinstance(updated_at, datetime):
|
||||
return row, batch
|
||||
if not executed_batch_runner_lost(batch.status, updated_at):
|
||||
return row, batch
|
||||
runner: Final = _litellm_executed_batch_runner(llm_router, proxy_logging_obj)
|
||||
return row, await runner.fail_abandoned(batch, user_api_key_dict)
|
||||
|
||||
|
||||
async def _raise_when_input_file_must_be_managed(model: str, credentials: Mapping[str, object]) -> None:
|
||||
if await litellm_executed_provider_for(credentials) is None:
|
||||
return
|
||||
raise batch_error(
|
||||
400,
|
||||
f"Batches for {model} run inside LiteLLM, so the input file must be a LiteLLM managed file: "
|
||||
f"{LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE}",
|
||||
)
|
||||
|
||||
|
||||
def _raise_not_found_when_openai_fallback_unservable(
|
||||
|
|
@ -101,6 +183,24 @@ async def _resolve_managed_input_file_storage_url(input_file_id: str) -> "str |
|
|||
return db_file.storage_url or None
|
||||
|
||||
|
||||
async def _create_provider_batch_for_managed_file(
|
||||
llm_router: Router,
|
||||
create_batch_data: LiteLLMBatchCreateRequest,
|
||||
input_file_id: str,
|
||||
unified_file_id: str,
|
||||
) -> LiteLLMBatch:
|
||||
resolved_storage_url: Final = await _resolve_managed_input_file_storage_url(input_file_id)
|
||||
request: Final[LiteLLMBatchCreateRequest] = {
|
||||
**create_batch_data,
|
||||
"input_file_id": resolved_storage_url or input_file_id,
|
||||
"disable_fallbacks": True,
|
||||
}
|
||||
response: Final = await llm_router.acreate_batch(**request)
|
||||
response.input_file_id = input_file_id
|
||||
response._hidden_params["unified_file_id"] = unified_file_id
|
||||
return response
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{provider}/v1/batches",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
@ -296,24 +396,35 @@ async def create_batch(
|
|||
await authorize_model_for_key(model_id=model, llm_router=llm_router, user_api_key_dict=user_api_key_dict)
|
||||
_create_batch_data["model"] = model
|
||||
|
||||
resolved_storage_url: Final = await _resolve_managed_input_file_storage_url(input_file_id)
|
||||
if resolved_storage_url is not None:
|
||||
_create_batch_data["input_file_id"] = resolved_storage_url
|
||||
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "LLM Router not initialized. Ensure models added to proxy."},
|
||||
)
|
||||
|
||||
_create_batch_data.update(disable_fallbacks=True) # pyright: ignore[reportCallIssue] # router flag
|
||||
response = await llm_router.acreate_batch(**_create_batch_data)
|
||||
response.input_file_id = input_file_id
|
||||
response._hidden_params["unified_file_id"] = unified_file_id
|
||||
executed_provider: Final = await resolve_litellm_executed_provider(
|
||||
llm_router, model, user_api_key_dict.team_id
|
||||
)
|
||||
response = (
|
||||
await _litellm_executed_batch_runner(llm_router, proxy_logging_obj).create(
|
||||
create_request=_create_batch_data,
|
||||
unified_input_file_id=input_file_id,
|
||||
model=model,
|
||||
provider=executed_provider,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_tags=_request_tags(_create_batch_data),
|
||||
)
|
||||
if executed_provider is not None
|
||||
else await _create_provider_batch_for_managed_file(
|
||||
llm_router, _create_batch_data, input_file_id, unified_file_id
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Check if model specified via header/query/body param
|
||||
model_param: Final = (
|
||||
data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model")
|
||||
_create_batch_data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
|
||||
# SCENARIO 2 & 3: Model from header/query OR custom_llm_provider fallback
|
||||
|
|
@ -325,6 +436,7 @@ async def create_batch(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
operation_context="batch creation",
|
||||
)
|
||||
await _raise_when_input_file_must_be_managed(model_param, credentials)
|
||||
|
||||
prepare_data_with_credentials(
|
||||
data=_create_batch_data,
|
||||
|
|
@ -486,23 +598,26 @@ async def retrieve_batch(
|
|||
managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files")
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
db_batch_object, response = await get_batch_from_database(
|
||||
executed_batch: Final = isinstance(unified_batch_id, str) and is_litellm_executed_batch(unified_batch_id)
|
||||
db_batch_object, response = await _batch_from_database(
|
||||
batch_id=batch_id,
|
||||
unified_batch_id=unified_batch_id,
|
||||
executed_batch=executed_batch,
|
||||
managed_files_obj=managed_files_obj,
|
||||
prisma_client=prisma_client,
|
||||
verbose_proxy_logger=verbose_proxy_logger,
|
||||
llm_router=llm_router,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
if executed_batch and response is None:
|
||||
raise batch_error(404, f"No batch found with id '{batch_id}'.")
|
||||
|
||||
# If batch is in a terminal state, return immediately.
|
||||
# Include "complete" (DB-normalized form of "completed").
|
||||
if response is not None and response.status in [
|
||||
"completed",
|
||||
"complete",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"expired",
|
||||
]:
|
||||
if response is not None and (
|
||||
response.status in ("completed", "complete", "failed", "cancelled", "expired") or executed_batch
|
||||
):
|
||||
# Call hooks and return
|
||||
response = await proxy_logging_obj.post_call_success_hook(
|
||||
data=data, user_api_key_dict=user_api_key_dict, response=response
|
||||
|
|
@ -978,6 +1093,17 @@ async def cancel_batch(
|
|||
proxy_config=proxy_config,
|
||||
)
|
||||
|
||||
unified_model_id: Final = get_model_id_from_unified_batch_id(unified_batch_id) if unified_batch_id else None
|
||||
if unified_model_id is not None:
|
||||
resolved_unified_model: Final = (
|
||||
llm_router.resolve_model_name_from_model_id(unified_model_id) if llm_router is not None else None
|
||||
)
|
||||
await authorize_model_for_key(
|
||||
model_id=resolved_unified_model or unified_model_id,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
# SCENARIO 1: Batch ID is encoded with model info
|
||||
if model_from_id is not None:
|
||||
credentials: Final = await get_authorized_credentials_for_model(
|
||||
|
|
@ -1009,6 +1135,12 @@ async def cancel_batch(
|
|||
)
|
||||
|
||||
# SCENARIO 2: target_model_names based routing
|
||||
elif unified_batch_id and is_litellm_executed_batch(unified_batch_id):
|
||||
if llm_router is None:
|
||||
raise batch_error(500, "LLM Router not initialized. Ensure models added to proxy.")
|
||||
response = await _litellm_executed_batch_runner( # rebind-ok: each cancel path sets the route's response
|
||||
llm_router, proxy_logging_obj
|
||||
).cancel(batch_id, user_api_key_dict)
|
||||
elif unified_batch_id:
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
|
|
@ -1022,11 +1154,6 @@ async def cancel_batch(
|
|||
status_code=400,
|
||||
detail={"error": "Invalid LiteLLM managed batch ID. Missing model_id."},
|
||||
)
|
||||
await authorize_model_for_key(
|
||||
model_id=llm_router.resolve_model_name_from_model_id(model_id_from_batch) or model_id_from_batch,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
data["model"] = model_id_from_batch
|
||||
data["batch_id"] = get_batch_id_from_unified_batch_id(unified_batch_id)
|
||||
response = await llm_router.acancel_batch(**data)
|
||||
|
|
|
|||
716
litellm/proxy/batches_endpoints/litellm_executed_batches.py
Normal file
716
litellm/proxy/batches_endpoints/litellm_executed_batches.py
Normal file
|
|
@ -0,0 +1,716 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import pairwise
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
import httpx
|
||||
from openai.types.batch import Errors
|
||||
from openai.types.batch_error import BatchError
|
||||
from openai.types.batch_request_counts import BatchRequestCounts
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid as uuid_module
|
||||
from litellm.constants import LITELLM_EXECUTED_BATCH_CONCURRENCY
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME
|
||||
from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend
|
||||
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.models.managed_files import LiteLLM_ManagedFileTable
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import is_request_body_safe
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import LITELLM_EXECUTED_BATCH_ID_PREFIX
|
||||
from litellm.proxy.openai_files_endpoints.storage_backend_service import StorageBackendFileService
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.managed_batch_repository import ManagedBatchRepository
|
||||
from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose
|
||||
from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import types as prisma_types
|
||||
|
||||
from litellm.router import Router
|
||||
|
||||
BatchEndpoint: TypeAlias = Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"]
|
||||
BatchStatus: TypeAlias = Literal[
|
||||
"in_progress", "finalizing", "completed", "failed", "cancelling", "cancelled", "expired"
|
||||
]
|
||||
TERMINAL_BATCH_STATUSES: Final[frozenset[str]] = frozenset({"completed", "failed", "cancelled", "expired"})
|
||||
_STOP_STATUSES: Final[frozenset[str]] = TERMINAL_BATCH_STATUSES | frozenset({"cancelling"})
|
||||
_BATCH_ENDPOINT_ADAPTER: Final[TypeAdapter[BatchEndpoint]] = TypeAdapter(BatchEndpoint)
|
||||
_CANCEL_POLL_SECONDS: Final = 1.0
|
||||
_HEARTBEAT_SECONDS: Final = 30.0
|
||||
_STALE_AFTER_SECONDS: Final = 180.0
|
||||
_FILES_API_PROBE_TIMEOUT_SECONDS: Final = 5.0
|
||||
_COMPLETION_WINDOW_SECONDS: Final = 24 * 60 * 60
|
||||
_RUNNER_LOST_MESSAGE: Final = "the proxy replica running this batch stopped before it finished; resubmit the batch"
|
||||
_EXPIRED_MESSAGE: Final = "This request could not be executed before the completion window expired."
|
||||
_ROUTER_METHODS: Final[Mapping[BatchEndpoint, str]] = MappingProxyType(
|
||||
{
|
||||
"/v1/chat/completions": "acompletion",
|
||||
"/v1/completions": "atext_completion",
|
||||
"/v1/embeddings": "aembedding",
|
||||
"/v1/responses": "aresponses",
|
||||
}
|
||||
)
|
||||
_CANCELLING_TRANSITIONS: Final[Mapping[BatchStatus, BatchStatus]] = MappingProxyType(
|
||||
{"completed": "cancelled", "expired": "cancelled", "in_progress": "cancelling", "finalizing": "cancelling"}
|
||||
)
|
||||
LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE: Final = (
|
||||
"upload it through POST /v1/files with purpose=batch and either the x-litellm-model header or the "
|
||||
"target_model_names form field naming the model, so LiteLLM keeps the file and runs the batch itself"
|
||||
)
|
||||
_RUNNING_BATCHES: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong references keep running batch tasks alive
|
||||
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
class _ErrorDetail(TypedDict):
|
||||
message: ReadOnly[str]
|
||||
type: ReadOnly[str]
|
||||
param: ReadOnly[None]
|
||||
code: ReadOnly[None]
|
||||
|
||||
|
||||
class _ErrorBody(TypedDict):
|
||||
error: ReadOnly[_ErrorDetail]
|
||||
|
||||
|
||||
class _ResultResponse(TypedDict):
|
||||
status_code: ReadOnly[int]
|
||||
request_id: ReadOnly[str]
|
||||
body: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
class _LineError(TypedDict):
|
||||
code: ReadOnly[str]
|
||||
message: ReadOnly[str]
|
||||
|
||||
|
||||
class _ResultLine(TypedDict):
|
||||
id: ReadOnly[str]
|
||||
custom_id: ReadOnly[str]
|
||||
response: ReadOnly[_ResultResponse | None]
|
||||
error: ReadOnly[_LineError | None]
|
||||
|
||||
|
||||
class BatchInputLine(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
custom_id: str
|
||||
method: Literal["POST"]
|
||||
url: str
|
||||
body: Mapping[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InvalidBatchInput:
|
||||
line_number: int | None
|
||||
reason: str
|
||||
|
||||
def describe(self) -> str:
|
||||
return f"line {self.line_number}: {self.reason}" if self.line_number is not None else self.reason
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RowOutcome:
|
||||
custom_id: str
|
||||
status_code: int
|
||||
body: Mapping[str, object]
|
||||
succeeded: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExpiredRow:
|
||||
custom_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BatchRun:
|
||||
unified_batch_id: str
|
||||
llm_batch_id: str
|
||||
model: str
|
||||
endpoint: BatchEndpoint
|
||||
lines: tuple[BatchInputLine, ...]
|
||||
user_api_key_dict: UserAPIKeyAuth
|
||||
request_tags: tuple[str, ...]
|
||||
deadline: float
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ManagedBatchStore(Protocol):
|
||||
def get_unified_batch_id(self, batch_id: str, model_id: str) -> str: ...
|
||||
|
||||
async def get_unified_file_id(
|
||||
self, file_id: str, litellm_parent_otel_span: object | None = None
|
||||
) -> LiteLLM_ManagedFileTable | None: ...
|
||||
|
||||
async def store_unified_object_id(
|
||||
self,
|
||||
unified_object_id: str,
|
||||
file_object: LiteLLMBatch,
|
||||
litellm_parent_otel_span: object | None,
|
||||
model_object_id: str,
|
||||
file_purpose: Literal["batch", "fine-tune", "response"],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
request_tags: Sequence[str] | None = None,
|
||||
persist_attribution: bool = False,
|
||||
batch_processed: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class _StorageBackendFactory(Protocol):
|
||||
def __call__(self, backend_type: str, prisma_client: PrismaClient | None = None) -> BaseFileStorageBackend: ...
|
||||
|
||||
|
||||
class _ResultFileUploader(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
file_data: Mapping[str, object],
|
||||
target_storage: str,
|
||||
target_model_names: Sequence[str],
|
||||
purpose: OpenAIFilesPurpose,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
prisma_client: PrismaClient | None = None,
|
||||
) -> Awaitable[OpenAIFileObject]: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _RouterCall(Protocol):
|
||||
def __call__(self, **params: object) -> Awaitable[object]: ... # kwargs-ok: the request body is passed as keywords
|
||||
|
||||
|
||||
def litellm_executed_provider_of(credentials: Mapping[str, object]) -> str | None:
|
||||
explicit_provider: Final = credentials.get("custom_llm_provider")
|
||||
provider: Final = (
|
||||
explicit_provider if isinstance(explicit_provider, str) else _provider_of(credentials.get("model"))
|
||||
)
|
||||
return provider if provider in LITELLM_EXECUTED_BATCH_PROVIDERS else None
|
||||
|
||||
|
||||
class _HttpGetter(Protocol):
|
||||
async def get(
|
||||
self, url: str, *, headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None
|
||||
) -> httpx.Response: ...
|
||||
|
||||
|
||||
class FilesApiProbe(Protocol):
|
||||
async def __call__(self, api_base: str, api_key: str | None) -> bool: ...
|
||||
|
||||
|
||||
class BodyRejection(Protocol):
|
||||
def __call__(self, body: Mapping[str, object], /) -> str | None: ...
|
||||
|
||||
|
||||
async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_client: _HttpGetter | None = None) -> bool:
|
||||
client: Final = http_client or get_async_httpx_client(llm_provider=LlmProviders.HOSTED_VLLM)
|
||||
try:
|
||||
response: Final = await client.get(
|
||||
f"{api_base.rstrip('/')}/files",
|
||||
headers=(
|
||||
{"Authorization": f"Bearer {api_key}"} # mutable-ok: AsyncHTTPHandler.get wants a plain dict
|
||||
if api_key
|
||||
else None
|
||||
),
|
||||
timeout=_FILES_API_PROBE_TIMEOUT_SECONDS,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
return response.status_code == httpx.codes.NOT_FOUND
|
||||
|
||||
|
||||
def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
|
||||
model: Final = credentials.get("model")
|
||||
api_base: Final = credentials.get("api_base")
|
||||
api_key: Final = credentials.get("api_key")
|
||||
if not isinstance(model, str):
|
||||
return None
|
||||
try:
|
||||
_, _, resolved_api_key, resolved_api_base = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=provider,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
api_key=api_key if isinstance(api_key, str) else None,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # get_llm_provider raises on a model it cannot map, which means nothing to probe
|
||||
return None
|
||||
return None if resolved_api_base is None else (resolved_api_base, resolved_api_key)
|
||||
|
||||
|
||||
async def litellm_executed_provider_for(
|
||||
credentials: Mapping[str, object], lacks_files_api: FilesApiProbe = upstream_lacks_files_api
|
||||
) -> str | None:
|
||||
provider: Final = litellm_executed_provider_of(credentials)
|
||||
if provider is None:
|
||||
return None
|
||||
upstream: Final = _upstream_of(credentials, provider)
|
||||
if upstream is None:
|
||||
return None
|
||||
return provider if await lacks_files_api(*upstream) else None
|
||||
|
||||
|
||||
async def resolve_litellm_executed_provider(
|
||||
llm_router: "Router",
|
||||
model: str,
|
||||
team_id: str | None,
|
||||
lacks_files_api: FilesApiProbe = upstream_lacks_files_api,
|
||||
) -> str | None:
|
||||
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model, team_id=team_id)
|
||||
return None if credentials is None else await litellm_executed_provider_for(credentials, lacks_files_api)
|
||||
|
||||
|
||||
def _provider_of(model: object) -> str | None:
|
||||
if not isinstance(model, str):
|
||||
return None
|
||||
try:
|
||||
return litellm.get_llm_provider(model=model)[1]
|
||||
except Exception: # noqa: BLE001 # get_llm_provider raises on an unknown model, which means no provider
|
||||
return None
|
||||
|
||||
|
||||
def _validation_reason(error: ValidationError) -> str:
|
||||
return "; ".join(
|
||||
f"{'.'.join(str(part) for part in item['loc'])}: {item['msg']}" if item["loc"] else item["msg"]
|
||||
for item in error.errors()
|
||||
)
|
||||
|
||||
|
||||
def _accept_every_body(_body: Mapping[str, object]) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_line(
|
||||
line_number: int, raw: bytes, endpoint: BatchEndpoint, reject_body: BodyRejection
|
||||
) -> BatchInputLine | InvalidBatchInput:
|
||||
try:
|
||||
line: Final = BatchInputLine.model_validate_json(raw)
|
||||
except ValidationError as e:
|
||||
return InvalidBatchInput(line_number, _validation_reason(e))
|
||||
if line.url != endpoint:
|
||||
return InvalidBatchInput(line_number, f"url {line.url!r} does not match the batch endpoint {endpoint!r}")
|
||||
if line.body.get("stream"):
|
||||
return InvalidBatchInput(line_number, "streaming requests are not supported in a batch")
|
||||
rejection: Final = reject_body(line.body)
|
||||
if rejection is not None:
|
||||
return InvalidBatchInput(line_number, rejection)
|
||||
return line
|
||||
|
||||
|
||||
def parse_batch_input(
|
||||
content: bytes, endpoint: BatchEndpoint, reject_body: BodyRejection = _accept_every_body
|
||||
) -> tuple[BatchInputLine, ...] | InvalidBatchInput:
|
||||
raw_lines: Final = tuple((number, raw) for number, raw in enumerate(content.splitlines(), start=1) if raw.strip())
|
||||
if not raw_lines:
|
||||
return InvalidBatchInput(None, "the input file has no requests")
|
||||
parsed: Final = tuple(_parse_line(number, raw, endpoint, reject_body) for number, raw in raw_lines)
|
||||
first_invalid: Final = next((item for item in parsed if isinstance(item, InvalidBatchInput)), None)
|
||||
if first_invalid is not None:
|
||||
return first_invalid
|
||||
lines: Final = tuple(item for item in parsed if isinstance(item, BatchInputLine))
|
||||
custom_ids: Final = sorted(line.custom_id for line in lines)
|
||||
duplicate: Final = next((first for first, second in pairwise(custom_ids) if first == second), None)
|
||||
if duplicate is not None:
|
||||
return InvalidBatchInput(None, f"custom_id {duplicate!r} is used more than once")
|
||||
return lines
|
||||
|
||||
|
||||
def batch_error(status_code: int, message: str) -> ProxyException:
|
||||
error_type: Final = "invalid_request_error" if status_code < 500 else ProxyErrorTypes.internal_server_error.value
|
||||
return ProxyException(message=message, type=error_type, param=None, code=status_code)
|
||||
|
||||
|
||||
def _validate_endpoint(endpoint: object) -> BatchEndpoint:
|
||||
try:
|
||||
return _BATCH_ENDPOINT_ADAPTER.validate_python(endpoint)
|
||||
except ValidationError:
|
||||
raise batch_error(400, f"endpoint {endpoint!r} is not supported for a LiteLLM-executed batch")
|
||||
|
||||
|
||||
def _status_code_of(error: Exception) -> int:
|
||||
status_code: Final[object] = getattr(error, "status_code", None)
|
||||
return status_code if isinstance(status_code, int) else 500
|
||||
|
||||
|
||||
def _error_body(error: Exception) -> _ErrorBody:
|
||||
body: Final[_ErrorBody] = {
|
||||
"error": {"message": str(error), "type": type(error).__name__, "param": None, "code": None}
|
||||
}
|
||||
return body
|
||||
|
||||
|
||||
def _line_response(outcome: RowOutcome | ExpiredRow) -> _ResultResponse | None:
|
||||
if isinstance(outcome, ExpiredRow):
|
||||
return None
|
||||
response: Final[_ResultResponse] = {
|
||||
"status_code": outcome.status_code,
|
||||
"request_id": f"req_{uuid_module.uuid4().hex[:24]}",
|
||||
"body": outcome.body,
|
||||
}
|
||||
return response
|
||||
|
||||
|
||||
def _line_error(outcome: RowOutcome | ExpiredRow) -> _LineError | None:
|
||||
if isinstance(outcome, RowOutcome):
|
||||
return None
|
||||
error: Final[_LineError] = {"code": "batch_expired", "message": _EXPIRED_MESSAGE}
|
||||
return error
|
||||
|
||||
|
||||
def _result_line(outcome: RowOutcome | ExpiredRow) -> _ResultLine:
|
||||
line: Final[_ResultLine] = {
|
||||
"id": f"batch_req_{uuid_module.uuid4().hex[:24]}",
|
||||
"custom_id": outcome.custom_id,
|
||||
"response": _line_response(outcome),
|
||||
"error": _line_error(outcome),
|
||||
}
|
||||
return line
|
||||
|
||||
|
||||
def _dump(response: object) -> Mapping[str, object]:
|
||||
if isinstance(response, BaseModel):
|
||||
return response.model_dump(mode="json")
|
||||
raise TypeError(f"Batch rows must return a single response object, got {type(response).__name__}")
|
||||
|
||||
|
||||
def _resolve_transition(current_status: str, requested: BatchStatus) -> BatchStatus:
|
||||
if current_status != "cancelling":
|
||||
return requested
|
||||
return _CANCELLING_TRANSITIONS.get(requested, requested)
|
||||
|
||||
|
||||
def executed_batch_runner_lost(status: str, updated_at: datetime) -> bool:
|
||||
if status in TERMINAL_BATCH_STATUSES:
|
||||
return False
|
||||
return (datetime.now(timezone.utc) - updated_at).total_seconds() > _STALE_AFTER_SECONDS
|
||||
|
||||
|
||||
class _StopWatch:
|
||||
def __init__(self, load_status: Callable[[], Awaitable[str | None]], interval_seconds: float) -> None:
|
||||
self._load_status = load_status
|
||||
self._interval_seconds = interval_seconds
|
||||
self._checked_at = float("-inf")
|
||||
self._stopped = False
|
||||
|
||||
async def stopped(self) -> bool:
|
||||
if self._stopped:
|
||||
return True
|
||||
now: Final = time.monotonic()
|
||||
if now - self._checked_at < self._interval_seconds:
|
||||
return False
|
||||
self._checked_at = now
|
||||
self._stopped = await self._load_status() in _STOP_STATUSES
|
||||
return self._stopped
|
||||
|
||||
|
||||
class LiteLLMExecutedBatchRunner:
|
||||
def __init__(
|
||||
self,
|
||||
llm_router: "Router",
|
||||
prisma_client: PrismaClient,
|
||||
managed_files: ManagedBatchStore,
|
||||
batches: ManagedBatchRepository,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
general_settings: Mapping[str, object],
|
||||
concurrency: int = LITELLM_EXECUTED_BATCH_CONCURRENCY,
|
||||
heartbeat_seconds: float = _HEARTBEAT_SECONDS,
|
||||
completion_window_seconds: float = _COMPLETION_WINDOW_SECONDS,
|
||||
storage_backend_factory: _StorageBackendFactory = get_storage_backend,
|
||||
upload_result_file: _ResultFileUploader = StorageBackendFileService.upload_file_to_storage_backend,
|
||||
) -> None:
|
||||
self.llm_router = llm_router
|
||||
self.prisma_client = prisma_client
|
||||
self.managed_files = managed_files
|
||||
self.batches = batches
|
||||
self.proxy_logging_obj = proxy_logging_obj
|
||||
self.general_settings = general_settings
|
||||
self.concurrency = concurrency
|
||||
self.heartbeat_seconds = heartbeat_seconds
|
||||
self.completion_window_seconds = completion_window_seconds
|
||||
self.storage_backend_factory = storage_backend_factory
|
||||
self.upload_result_file = upload_result_file
|
||||
|
||||
async def create(
|
||||
self,
|
||||
create_request: LiteLLMBatchCreateRequest,
|
||||
unified_input_file_id: str,
|
||||
model: str,
|
||||
provider: str,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
request_tags: Sequence[str] | None,
|
||||
) -> LiteLLMBatch:
|
||||
endpoint: Final = _validate_endpoint(create_request.get("endpoint"))
|
||||
content: Final = await self._download_input(unified_input_file_id, user_api_key_dict)
|
||||
parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model))
|
||||
if isinstance(parsed, InvalidBatchInput):
|
||||
raise batch_error(400, f"Invalid batch input file: {parsed.describe()}")
|
||||
llm_batch_id: Final = f"{LITELLM_EXECUTED_BATCH_ID_PREFIX}{uuid_module.uuid4().hex}"
|
||||
model_id: Final = next(iter(self.llm_router.get_model_ids(model_name=model)), model)
|
||||
unified_batch_id: Final = self.managed_files.get_unified_batch_id(batch_id=llm_batch_id, model_id=model_id)
|
||||
now: Final = time.time()
|
||||
created_at: Final = int(now)
|
||||
batch: Final = LiteLLMBatch(
|
||||
id=unified_batch_id,
|
||||
object="batch",
|
||||
endpoint=endpoint,
|
||||
input_file_id=unified_input_file_id,
|
||||
completion_window="24h",
|
||||
status="validating",
|
||||
created_at=created_at,
|
||||
expires_at=created_at + int(self.completion_window_seconds),
|
||||
metadata=create_request.get("metadata"),
|
||||
model=model,
|
||||
request_counts=BatchRequestCounts(completed=0, failed=0, total=len(parsed)),
|
||||
)
|
||||
await self.managed_files.store_unified_object_id(
|
||||
unified_object_id=unified_batch_id,
|
||||
file_object=batch,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_object_id=llm_batch_id,
|
||||
file_purpose="batch",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_tags=request_tags,
|
||||
persist_attribution=True,
|
||||
batch_processed=True,
|
||||
)
|
||||
_record_batch_created(model, provider, user_api_key_dict)
|
||||
run: Final = _BatchRun(
|
||||
unified_batch_id=unified_batch_id,
|
||||
llm_batch_id=llm_batch_id,
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
lines=parsed,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_tags=tuple(request_tags or ()),
|
||||
deadline=now + self.completion_window_seconds,
|
||||
)
|
||||
task: Final = asyncio.create_task(self._run(run))
|
||||
_RUNNING_BATCHES.add(task)
|
||||
task.add_done_callback(_RUNNING_BATCHES.discard)
|
||||
return batch
|
||||
|
||||
async def cancel(self, unified_batch_id: str, user_api_key_dict: UserAPIKeyAuth) -> LiteLLMBatch:
|
||||
current: Final = await self.batches.load_batch(unified_batch_id)
|
||||
if current is None:
|
||||
raise batch_error(404, f"Batch {unified_batch_id} not found")
|
||||
if current.status in TERMINAL_BATCH_STATUSES:
|
||||
raise batch_error(400, f"Cannot cancel a batch with status '{current.status}'")
|
||||
if current.status == "cancelling":
|
||||
return current
|
||||
cancelling: Final = current.model_copy(
|
||||
update=MappingProxyType({"status": "cancelling", "cancelling_at": int(time.time())})
|
||||
)
|
||||
unchanged: Final[prisma_types.LiteLLM_ManagedObjectTableWhereInput] = {"status": current.status}
|
||||
if await self.batches.compare_and_set(cancelling, unchanged, user_api_key_dict.user_id):
|
||||
return cancelling
|
||||
return await self.cancel(unified_batch_id, user_api_key_dict)
|
||||
|
||||
async def fail_abandoned(self, batch: LiteLLMBatch, user_api_key_dict: UserAPIKeyAuth) -> LiteLLMBatch:
|
||||
error: Final = BatchError(message=_RUNNER_LOST_MESSAGE, code="runner_lost")
|
||||
errors: Final = Errors(data=[error], object="list") # mutable-ok: Errors.data is typed as a list
|
||||
failed: Final = batch.model_copy(
|
||||
update=MappingProxyType({"status": "failed", "failed_at": int(time.time()), "errors": errors})
|
||||
)
|
||||
untouched: Final[prisma_types.DateTimeFilter] = {
|
||||
"lt": datetime.now(timezone.utc) - timedelta(seconds=_STALE_AFTER_SECONDS)
|
||||
}
|
||||
still_abandoned: Final[prisma_types.LiteLLM_ManagedObjectTableWhereInput] = {
|
||||
"status": batch.status,
|
||||
"updated_at": untouched,
|
||||
}
|
||||
if await self.batches.compare_and_set(failed, still_abandoned, user_api_key_dict.user_id):
|
||||
return failed
|
||||
return await self.batches.load_batch(batch.id) or batch
|
||||
|
||||
def _body_rejection(self, model: str) -> BodyRejection:
|
||||
def reject(body: Mapping[str, object]) -> str | None:
|
||||
try:
|
||||
is_request_body_safe(
|
||||
request_body=dict(body), # mutable-ok: is_request_body_safe takes a dict
|
||||
general_settings=dict(self.general_settings), # mutable-ok: is_request_body_safe takes a dict
|
||||
llm_router=self.llm_router,
|
||||
model=model,
|
||||
)
|
||||
except ValueError as e:
|
||||
return str(e)
|
||||
return None
|
||||
|
||||
return reject
|
||||
|
||||
async def _download_input(self, unified_input_file_id: str, user_api_key_dict: UserAPIKeyAuth) -> bytes:
|
||||
stored: Final = await self.managed_files.get_unified_file_id(
|
||||
unified_input_file_id, litellm_parent_otel_span=user_api_key_dict.parent_otel_span
|
||||
)
|
||||
if stored is None or not stored.storage_backend or not stored.storage_url:
|
||||
raise batch_error(
|
||||
400,
|
||||
f"LiteLLM does not hold the content of input file {unified_input_file_id}: "
|
||||
f"{LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE}",
|
||||
)
|
||||
try:
|
||||
backend: Final = self.storage_backend_factory(stored.storage_backend, prisma_client=self.prisma_client)
|
||||
return await backend.download_file(stored.storage_url)
|
||||
except ValueError as e:
|
||||
raise batch_error(400, str(e))
|
||||
|
||||
async def _run(self, run: _BatchRun) -> None:
|
||||
heartbeat: Final = asyncio.create_task(self._heartbeat(run))
|
||||
try:
|
||||
await self._execute(run)
|
||||
except Exception as e: # noqa: BLE001 # whatever fails, the batch must end up marked failed
|
||||
verbose_proxy_logger.exception("LiteLLM-executed batch %s failed: %s", run.unified_batch_id, e)
|
||||
error: Final = BatchError(message=str(e), code="internal_error")
|
||||
errors: Final = Errors(data=[error], object="list") # mutable-ok: Errors.data is typed as a list
|
||||
try:
|
||||
await self._advance(run, "failed", MappingProxyType({"errors": errors}))
|
||||
except Exception as advance_error: # noqa: BLE001 # a failed status write is logged, never raised
|
||||
verbose_proxy_logger.exception(
|
||||
"LiteLLM-executed batch %s could not be marked failed: %s", run.unified_batch_id, advance_error
|
||||
)
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
|
||||
async def _heartbeat(self, run: _BatchRun) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(self.heartbeat_seconds)
|
||||
try:
|
||||
await self._touch(run)
|
||||
except Exception as e: # noqa: BLE001 # a missed beat is logged and the next one retries
|
||||
verbose_proxy_logger.warning("LiteLLM-executed batch %s heartbeat failed: %s", run.unified_batch_id, e)
|
||||
|
||||
async def _touch(self, run: _BatchRun) -> None:
|
||||
await self.batches.touch(run.unified_batch_id, run.user_api_key_dict.user_id)
|
||||
|
||||
async def _execute(self, run: _BatchRun) -> None:
|
||||
await self._advance(run, "in_progress")
|
||||
watch: Final = _StopWatch(lambda: self.batches.load_status(run.unified_batch_id), _CANCEL_POLL_SECONDS)
|
||||
semaphore: Final = asyncio.Semaphore(self.concurrency)
|
||||
results: Final = await asyncio.gather(*(self._run_row(run, line, watch, semaphore) for line in run.lines))
|
||||
outcomes: Final = tuple(outcome for outcome in results if outcome is not None)
|
||||
if await self._advance(run, "finalizing") is None:
|
||||
return
|
||||
succeeded: Final = tuple(
|
||||
outcome for outcome in outcomes if isinstance(outcome, RowOutcome) and outcome.succeeded
|
||||
)
|
||||
failed: Final = tuple(
|
||||
outcome for outcome in outcomes if isinstance(outcome, ExpiredRow) or not outcome.succeeded
|
||||
)
|
||||
output_file_id: Final = await self._upload_results(run, "output", succeeded)
|
||||
error_file_id: Final = await self._upload_results(run, "error", failed)
|
||||
request_counts: Final = BatchRequestCounts(completed=len(succeeded), failed=len(failed), total=len(run.lines))
|
||||
final_status: Final[BatchStatus] = (
|
||||
"expired" if any(isinstance(outcome, ExpiredRow) for outcome in outcomes) else "completed"
|
||||
)
|
||||
await self._advance(
|
||||
run,
|
||||
final_status,
|
||||
MappingProxyType(
|
||||
{"output_file_id": output_file_id, "error_file_id": error_file_id, "request_counts": request_counts}
|
||||
),
|
||||
)
|
||||
|
||||
async def _run_row(
|
||||
self, run: _BatchRun, line: BatchInputLine, watch: _StopWatch, semaphore: asyncio.Semaphore
|
||||
) -> RowOutcome | ExpiredRow | None:
|
||||
async with semaphore:
|
||||
if await watch.stopped():
|
||||
return None
|
||||
remaining: Final = run.deadline - time.time()
|
||||
if remaining <= 0:
|
||||
return ExpiredRow(custom_id=line.custom_id)
|
||||
try:
|
||||
return await asyncio.wait_for(self._row_outcome(run, line), timeout=remaining)
|
||||
except asyncio.TimeoutError:
|
||||
return ExpiredRow(custom_id=line.custom_id)
|
||||
|
||||
async def _row_outcome(self, run: _BatchRun, line: BatchInputLine) -> RowOutcome:
|
||||
try:
|
||||
body: Final = await self._dispatch(run, line)
|
||||
except Exception as e: # noqa: BLE001 # a provider error becomes the row's error line, never a crashed batch
|
||||
return RowOutcome(
|
||||
custom_id=line.custom_id, status_code=_status_code_of(e), body=_error_body(e), succeeded=False
|
||||
)
|
||||
return RowOutcome(custom_id=line.custom_id, status_code=200, body=body, succeeded=True)
|
||||
|
||||
async def _dispatch(self, run: _BatchRun, line: BatchInputLine) -> Mapping[str, object]:
|
||||
params: Final = MappingProxyType(
|
||||
{**line.body, "model": run.model, "metadata": self._row_metadata(run), "disable_fallbacks": True}
|
||||
)
|
||||
return _dump(await self._router_call(run.endpoint)(**params))
|
||||
|
||||
def _router_call(self, endpoint: BatchEndpoint) -> _RouterCall:
|
||||
method: Final[object] = getattr(self.llm_router, _ROUTER_METHODS[endpoint], None)
|
||||
if not isinstance(method, _RouterCall):
|
||||
raise TypeError(f"the router has no callable for {endpoint}")
|
||||
return method
|
||||
|
||||
def _row_metadata(self, run: _BatchRun) -> dict[str, object]: # mutable-ok: router updates metadata in place
|
||||
return { # mutable-ok: the router updates request metadata in place
|
||||
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(run.user_api_key_dict),
|
||||
"user_api_key": run.user_api_key_dict.api_key,
|
||||
"user_api_end_user_max_budget": run.user_api_key_dict.end_user_max_budget,
|
||||
"tags": list(run.request_tags), # mutable-ok: litellm types request tags as a list
|
||||
"batch_id": run.unified_batch_id,
|
||||
}
|
||||
|
||||
async def _upload_results(
|
||||
self, run: _BatchRun, kind: Literal["output", "error"], outcomes: Sequence[RowOutcome | ExpiredRow]
|
||||
) -> str | None:
|
||||
if not outcomes:
|
||||
return None
|
||||
content: Final = "".join(f"{json.dumps(_result_line(outcome))}\n" for outcome in outcomes).encode()
|
||||
file_data: Final[ExtractedFileData] = {
|
||||
"filename": f"{run.llm_batch_id}_{kind}.jsonl",
|
||||
"content": content,
|
||||
"content_type": "application/jsonl",
|
||||
"headers": _NO_HEADERS,
|
||||
}
|
||||
file_object: Final = await self.upload_result_file(
|
||||
file_data=file_data,
|
||||
target_storage=LITELLM_DB_STORAGE_BACKEND_NAME,
|
||||
target_model_names=(run.model,),
|
||||
purpose="batch_output",
|
||||
proxy_logging_obj=self.proxy_logging_obj,
|
||||
user_api_key_dict=run.user_api_key_dict,
|
||||
prisma_client=self.prisma_client,
|
||||
)
|
||||
return file_object.id
|
||||
|
||||
async def _advance(
|
||||
self, run: _BatchRun, requested: BatchStatus, fields: Mapping[str, object] = _NO_FIELDS
|
||||
) -> BatchStatus | None:
|
||||
current: Final = await self.batches.load_batch(run.unified_batch_id)
|
||||
if current is None:
|
||||
raise RuntimeError(f"Batch {run.unified_batch_id} is no longer stored")
|
||||
if current.status in TERMINAL_BATCH_STATUSES:
|
||||
return None
|
||||
status: Final = _resolve_transition(current.status, requested)
|
||||
updated: Final = current.model_copy(
|
||||
update=MappingProxyType({**fields, "status": status, f"{status}_at": int(time.time())})
|
||||
)
|
||||
unchanged: Final[prisma_types.LiteLLM_ManagedObjectTableWhereInput] = {"status": current.status}
|
||||
if await self.batches.compare_and_set(updated, unchanged, run.user_api_key_dict.user_id):
|
||||
return status
|
||||
return await self._advance(run, requested, fields)
|
||||
|
||||
|
||||
def _record_batch_created(model: str, provider: str, user_api_key_dict: UserAPIKeyAuth) -> None:
|
||||
prometheus_logger: Final = PrometheusLogger.get_instance()
|
||||
if prometheus_logger is None:
|
||||
return
|
||||
prometheus_logger.record_managed_batch_created(
|
||||
model=model,
|
||||
api_provider=provider,
|
||||
user=user_api_key_dict.user_id or "",
|
||||
user_email=user_api_key_dict.user_email or "",
|
||||
api_key_alias=user_api_key_dict.key_alias or "",
|
||||
)
|
||||
|
|
@ -32,6 +32,7 @@ import unicodedata
|
|||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable, Mapping
|
||||
from math import isfinite
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import IO, Final, NamedTuple, Protocol
|
||||
|
|
@ -43,6 +44,7 @@ FETCH_TIMEOUT_SECONDS: Final = 3
|
|||
BAR_WIDTH: Final = 24
|
||||
BAR_FULL: Final = "\u2588"
|
||||
BAR_EMPTY: Final = "\u2591"
|
||||
SEPARATOR: Final = " \u00b7 "
|
||||
TRANSCRIPT_SCAN_LIMIT_BYTES: Final = 4 * 1024 * 1024
|
||||
CLAUDE_BASE_URL_ENV_KEYS: Final = ("ANTHROPIC_BASE_URL",)
|
||||
CLAUDE_API_KEY_ENV_KEYS: Final = ("ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY")
|
||||
|
|
@ -63,8 +65,11 @@ class Session(NamedTuple):
|
|||
router_name: str
|
||||
last_model: str
|
||||
spend: float
|
||||
baseline_spend: float
|
||||
baseline_spend: float | None
|
||||
baseline_model: str | None
|
||||
turns: int | None = None
|
||||
savings_estimated_turns: int | None = None
|
||||
savings_estimated_actual_spend: float | None = None
|
||||
|
||||
|
||||
class Credentials(NamedTuple):
|
||||
|
|
@ -205,17 +210,38 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None:
|
|||
router_name: Final = printable(payload.get("router_name"))
|
||||
last_model: Final = printable(payload.get("last_model"))
|
||||
spend: Final = payload.get("spend")
|
||||
baseline_spend: Final = payload.get("baseline_spend")
|
||||
baseline_spend: Final = payload.get("savings_estimated_baseline_spend", payload.get("baseline_spend"))
|
||||
turns: Final = payload.get("turns")
|
||||
estimated_turns: Final = payload.get("savings_estimated_turns")
|
||||
estimated_actual: Final = payload.get("savings_estimated_actual_spend")
|
||||
if not router_name or not last_model:
|
||||
return None
|
||||
if not isinstance(spend, (int, float)) or not isinstance(baseline_spend, (int, float)):
|
||||
if not isinstance(spend, (int, float)) or isinstance(spend, bool) or not isfinite(spend):
|
||||
return None
|
||||
if baseline_spend is not None and (
|
||||
not isinstance(baseline_spend, (int, float)) or isinstance(baseline_spend, bool) or not isfinite(baseline_spend)
|
||||
):
|
||||
return None
|
||||
return Session(
|
||||
router_name=router_name,
|
||||
last_model=last_model,
|
||||
spend=float(spend),
|
||||
baseline_spend=float(baseline_spend),
|
||||
baseline_spend=float(baseline_spend) if baseline_spend is not None else None,
|
||||
baseline_model=printable(payload.get("baseline_model")) or None,
|
||||
turns=turns if isinstance(turns, int) and not isinstance(turns, bool) and turns >= 0 else None,
|
||||
savings_estimated_turns=(
|
||||
estimated_turns
|
||||
if isinstance(estimated_turns, int) and not isinstance(estimated_turns, bool) and estimated_turns >= 0
|
||||
else (0 if estimated_turns is not None else None)
|
||||
),
|
||||
savings_estimated_actual_spend=(
|
||||
float(estimated_actual)
|
||||
if isinstance(estimated_actual, (int, float))
|
||||
and not isinstance(estimated_actual, bool)
|
||||
and isfinite(estimated_actual)
|
||||
and estimated_actual >= 0
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -314,15 +340,36 @@ def render(model: str, session: Session | None, config_dir: Path, use_color: boo
|
|||
return f"{code}{text}{RESET}" if use_color else text
|
||||
|
||||
routed: Final = paint(BOLD, f"Routed to: {model}")
|
||||
if session is None or session.baseline_model is None or session.baseline_spend <= 0:
|
||||
if session is None:
|
||||
return routed
|
||||
if session.savings_estimated_turns == 0 or session.baseline_spend is None:
|
||||
return f"{routed}{SEPARATOR}Savings unavailable"
|
||||
if session.baseline_model is None or session.baseline_spend <= 0:
|
||||
return routed
|
||||
if session.savings_estimated_turns is not None and (
|
||||
session.savings_estimated_actual_spend is None
|
||||
or session.turns is None
|
||||
or session.savings_estimated_turns > session.turns
|
||||
):
|
||||
return f"{routed}{SEPARATOR}Savings unavailable"
|
||||
compared_spend: Final = (
|
||||
session.savings_estimated_actual_spend
|
||||
if session.savings_estimated_turns is not None and session.savings_estimated_actual_spend is not None
|
||||
else session.spend
|
||||
)
|
||||
coverage: Final = (
|
||||
f"{SEPARATOR}{session.savings_estimated_turns} of {session.turns} turns estimated"
|
||||
if session.savings_estimated_turns is not None
|
||||
else ""
|
||||
)
|
||||
reference: Final = baseline_label(session.baseline_model, config_dir)
|
||||
pct: Final = (session.baseline_spend - session.spend) / session.baseline_spend * 100
|
||||
delta: Final = paint(LITELLM_COLOR, f"{'-' if pct >= 0 else '+'}{abs(round(pct))}% vs {reference}")
|
||||
peak: Final = max(session.spend, session.baseline_spend)
|
||||
pct: Final = round((session.baseline_spend - compared_spend) / session.baseline_spend * 100)
|
||||
sign: Final = "-" if pct > 0 else "+" if pct < 0 else ""
|
||||
delta: Final = paint(LITELLM_COLOR, f"{sign}{abs(pct)}% vs {reference}")
|
||||
peak: Final = max(compared_spend, session.baseline_spend)
|
||||
label_width: Final = max(_display_width(session.router_name), _display_width(reference))
|
||||
rows: Final = (
|
||||
(session.router_name, session.spend, LITELLM_COLOR),
|
||||
(session.router_name, compared_spend, LITELLM_COLOR),
|
||||
(reference, session.baseline_spend, BASELINE_COLOR),
|
||||
)
|
||||
lines: Final = (
|
||||
|
|
@ -331,7 +378,7 @@ def render(model: str, session: Session | None, config_dir: Path, use_color: boo
|
|||
f"{paint(DIM, f'${amount:.2f}')}"
|
||||
for label, amount, color in rows
|
||||
)
|
||||
return "\n".join((f"{routed} {delta}", *lines))
|
||||
return "\n".join((f"{routed} {delta}{coverage}", *lines))
|
||||
|
||||
|
||||
def color_enabled(env: Mapping[str, str]) -> bool:
|
||||
|
|
|
|||
|
|
@ -3749,6 +3749,14 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"async_streaming_data_generator: error closing response stream: %s",
|
||||
e,
|
||||
)
|
||||
logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
if (
|
||||
not stream_completed
|
||||
and isinstance(logging_obj, LiteLLMLoggingObj)
|
||||
and logging_obj.baseline_cache_context is not None
|
||||
and logging_obj.model_call_details.get("prompt_cache_response_complete") is not True
|
||||
):
|
||||
await logging_obj.invalidate_baseline_cache_estimate("incomplete_response", completed=True)
|
||||
|
||||
@staticmethod
|
||||
async def async_streaming_data_generator(
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Final, NamedTuple
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
|
||||
from litellm.proxy.db.create_views import SupportsExecuteRaw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy._types import SpendLogsPayload
|
||||
|
|
@ -75,6 +76,9 @@ SELECT
|
|||
COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens,
|
||||
COALESCE(SUM(spend), 0)::float8 AS spend,
|
||||
COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
|
||||
COALESCE(SUM(savings_estimated_turns), 0)::int AS savings_estimated_turns,
|
||||
COALESCE(SUM(savings_estimated_actual_spend), 0)::float8 AS savings_estimated_actual_spend,
|
||||
COALESCE(SUM(savings_estimated_saved_spend), 0)::float8 AS savings_estimated_saved_spend,
|
||||
COALESCE(SUM(classifier_cost), 0)::float8 AS classifier_cost,
|
||||
COALESCE(SUM(classifier_cost_recorded_turns), 0)::int AS classifier_cost_recorded_turns,
|
||||
COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
|
||||
|
|
@ -104,6 +108,9 @@ class AutoRouterTurnTransaction:
|
|||
cache_touched: bool
|
||||
tier: str | None = None
|
||||
baseline_model: str | None = None
|
||||
savings_estimated_turns: int = 0
|
||||
savings_estimated_actual_spend: float = 0.0
|
||||
savings_estimated_saved_spend: float = 0.0
|
||||
|
||||
|
||||
class TurnCacheFacts(NamedTuple):
|
||||
|
|
@ -215,13 +222,18 @@ def build_autorouter_turn_transaction(
|
|||
turn_at: Final = _turn_time_utc(str(payload.get("startTime") or ""))
|
||||
if turn_at is None:
|
||||
return None
|
||||
from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision
|
||||
from litellm.proxy.spend_tracking.savings import (
|
||||
classifier_cost_from_decision,
|
||||
recorded_estimated_autorouter_savings,
|
||||
)
|
||||
|
||||
usage_object_raw: Final = metadata.get("usage_object")
|
||||
cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None)
|
||||
tier_raw: Final = routing_decision.get("tier")
|
||||
baseline_raw: Final = routing_decision.get("savings_baseline_model")
|
||||
classifier_cost: Final = classifier_cost_from_decision(routing_decision)
|
||||
actual_spend: Final = float(payload.get("spend") or 0.0) + (classifier_cost or 0.0)
|
||||
estimated_savings: Final = recorded_estimated_autorouter_savings(metadata)
|
||||
return AutoRouterTurnTransaction(
|
||||
api_key=api_key,
|
||||
session_id=bounded_session_id(session_id),
|
||||
|
|
@ -232,13 +244,16 @@ def build_autorouter_turn_transaction(
|
|||
model=model,
|
||||
turn_at=turn_at,
|
||||
total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
|
||||
spend=float(payload.get("spend") or 0.0) + (classifier_cost or 0.0),
|
||||
spend=actual_spend,
|
||||
saved_spend=saved_spend,
|
||||
classifier_cost=classifier_cost or 0.0,
|
||||
covered=cache.covered,
|
||||
cache_hit=cache.read_tokens > 0,
|
||||
cache_ttl_seconds=cache.write_ttl_seconds,
|
||||
cache_touched=cache.touched,
|
||||
savings_estimated_turns=int(estimated_savings is not None),
|
||||
savings_estimated_actual_spend=actual_spend if estimated_savings is not None else 0.0,
|
||||
savings_estimated_saved_spend=estimated_savings if estimated_savings is not None else 0.0,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -263,6 +278,10 @@ _BASELINE: Final = f"{_p('baseline_model')}::text"
|
|||
_BASELINE_DELTA: Final = (
|
||||
f"(CASE WHEN {_BASELINE} IS NULL THEN '{{}}'::jsonb ELSE jsonb_build_object({_BASELINE}, 1) END)"
|
||||
)
|
||||
_ESTIMATED_BASELINE: Final = f"{_p('savings_estimated_turns')}::int = 1 AND {_BASELINE} IS NOT NULL"
|
||||
_ESTIMATED_BASELINE_DELTA: Final = (
|
||||
f"(CASE WHEN {_ESTIMATED_BASELINE} THEN jsonb_build_object({_BASELINE}, 1) ELSE '{{}}'::jsonb END)"
|
||||
)
|
||||
|
||||
_IN_ORDER: Final = f"{_TURN_AT}::timestamp >= t.last_turn_at"
|
||||
_SAME: Final = f"{_IN_ORDER} AND t.last_model = {_MODEL}"
|
||||
|
|
@ -281,7 +300,8 @@ INSERT INTO "LiteLLM_AutoRouterSession" AS t (
|
|||
same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
|
||||
return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
|
||||
ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns,
|
||||
baseline_models
|
||||
baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend,
|
||||
savings_estimated_baseline_models
|
||||
)
|
||||
VALUES (
|
||||
{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
|
||||
|
|
@ -292,13 +312,18 @@ VALUES (
|
|||
(CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_5M_SECONDS} THEN 1 ELSE 0 END),
|
||||
(CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_1H_SECONDS} THEN 1 ELSE 0 END),
|
||||
{_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8,
|
||||
{_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA}
|
||||
{_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA},
|
||||
{_p("savings_estimated_turns")}::int, {_p("savings_estimated_actual_spend")}::float8,
|
||||
{_p("savings_estimated_saved_spend")}::float8, {_ESTIMATED_BASELINE_DELTA}
|
||||
)
|
||||
ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
|
||||
turns = t.turns + 1,
|
||||
total_tokens = t.total_tokens + EXCLUDED.total_tokens,
|
||||
spend = t.spend + EXCLUDED.spend,
|
||||
saved_spend = t.saved_spend + EXCLUDED.saved_spend,
|
||||
savings_estimated_turns = t.savings_estimated_turns + EXCLUDED.savings_estimated_turns,
|
||||
savings_estimated_actual_spend = t.savings_estimated_actual_spend + EXCLUDED.savings_estimated_actual_spend,
|
||||
savings_estimated_saved_spend = t.savings_estimated_saved_spend + EXCLUDED.savings_estimated_saved_spend,
|
||||
classifier_cost = t.classifier_cost + EXCLUDED.classifier_cost,
|
||||
classifier_cost_recorded_turns = t.classifier_cost_recorded_turns + 1,
|
||||
covered_turns = t.covered_turns + EXCLUDED.covered_turns,
|
||||
|
|
@ -331,6 +356,10 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
|
|||
baseline_models = (CASE WHEN {_BASELINE} IS NOT NULL
|
||||
THEN t.baseline_models || jsonb_build_object({_BASELINE}, COALESCE((t.baseline_models ->> {_BASELINE})::int, 0) + 1)
|
||||
ELSE t.baseline_models END),
|
||||
savings_estimated_baseline_models = (CASE WHEN {_ESTIMATED_BASELINE}
|
||||
THEN t.savings_estimated_baseline_models || jsonb_build_object(
|
||||
{_BASELINE}, COALESCE((t.savings_estimated_baseline_models ->> {_BASELINE})::int, 0) + 1)
|
||||
ELSE t.savings_estimated_baseline_models END),
|
||||
first_turn_at = LEAST(t.first_turn_at, EXCLUDED.first_turn_at),
|
||||
last_turn_at = GREATEST(t.last_turn_at, EXCLUDED.last_turn_at)
|
||||
"""
|
||||
|
|
@ -348,6 +377,10 @@ def _upsert_params(transaction: AutoRouterTurnTransaction) -> tuple[str | float
|
|||
return tuple(_as_sql_param(getattr(transaction, name)) for name in _UPSERT_PARAM_FIELDS)
|
||||
|
||||
|
||||
async def write_autorouter_turn(db: SupportsExecuteRaw, transaction: AutoRouterTurnTransaction) -> None:
|
||||
await db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
|
||||
|
||||
|
||||
async def _upsert_turn_with_retry(
|
||||
prisma_client: PrismaClient,
|
||||
transaction: AutoRouterTurnTransaction,
|
||||
|
|
@ -355,7 +388,7 @@ async def _upsert_turn_with_retry(
|
|||
) -> None:
|
||||
for attempt in range(n_retry_times + 1):
|
||||
try:
|
||||
await prisma_client.db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
|
||||
await write_autorouter_turn(prisma_client.db, transaction)
|
||||
except DB_RETRY_SAFE_ERROR_TYPES:
|
||||
if attempt >= n_retry_times:
|
||||
raise
|
||||
|
|
|
|||
640
litellm/proxy/db/baseline_accounting.py
Normal file
640
litellm/proxy/db/baseline_accounting.py
Normal file
|
|
@ -0,0 +1,640 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Callable, Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from functools import reduce
|
||||
from itertools import groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.db.autorouter_session_rollup import (
|
||||
AutoRouterTurnTransaction,
|
||||
write_autorouter_turn,
|
||||
)
|
||||
from litellm.proxy.db.create_views import SupportsRawQueries
|
||||
from litellm.proxy.db.daily_spend_bulk_upsert import (
|
||||
DAILY_SPEND_TABLES,
|
||||
DailySpendEntity,
|
||||
SpendRow,
|
||||
build_bulk_upsert,
|
||||
merge_by_conflict_key,
|
||||
)
|
||||
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
|
||||
from litellm.proxy.spend_tracking.baseline_accounting import (
|
||||
BaselineEstimate,
|
||||
BaselineHistory,
|
||||
BaselineObservation,
|
||||
advance_baseline_history,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.savings import BaselineCosts, BaselineCostSnapshot, price_baseline_comparison
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
class DailyBaselineTarget(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
entity: DailySpendEntity
|
||||
entity_id: str | None
|
||||
|
||||
|
||||
class DailyBaselineAttribution(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
date: str
|
||||
api_key: str
|
||||
model: str | None = None
|
||||
custom_llm_provider: str | None = None
|
||||
model_group: str | None = None
|
||||
endpoint: str | None = None
|
||||
mcp_namespaced_tool_name: str | None = None
|
||||
targets: tuple[DailyBaselineTarget, ...] = ()
|
||||
|
||||
def adjustment(self, target: DailyBaselineTarget, savings_delta: float, request_id: str) -> SpendRow:
|
||||
table: Final = DAILY_SPEND_TABLES[target.entity]
|
||||
return MappingProxyType(
|
||||
{
|
||||
"date": self.date,
|
||||
"api_key": self.api_key,
|
||||
"model": self.model,
|
||||
"custom_llm_provider": self.custom_llm_provider,
|
||||
"model_group": self.model_group,
|
||||
"endpoint": self.endpoint,
|
||||
"mcp_namespaced_tool_name": self.mcp_namespaced_tool_name,
|
||||
table.entity_id_column: target.entity_id,
|
||||
"request_id": request_id,
|
||||
"autorouter_savings_spend": savings_delta,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class BaselineAccountingRecord(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
scope: str = Field(pattern=r"^autorouter-baseline:v3:[a-f0-9]{64}$")
|
||||
api_key: str = Field(min_length=1)
|
||||
session_id: str = Field(min_length=1, max_length=256)
|
||||
router_name: str = Field(min_length=1)
|
||||
baseline_model: str = Field(min_length=1)
|
||||
observation: BaselineObservation
|
||||
pricing: BaselineCostSnapshot
|
||||
turn: AutoRouterTurnTransaction | None
|
||||
daily: DailyBaselineAttribution | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def consistent_turn(self) -> Self:
|
||||
turn: Final = self.turn
|
||||
if turn is not None and (
|
||||
(turn.api_key, turn.session_id, turn.router_name, turn.baseline_model)
|
||||
!= (self.api_key, self.session_id, self.router_name, self.baseline_model)
|
||||
or turn.spend != self.pricing.actual_spend + self.pricing.classifier_cost
|
||||
or any(
|
||||
(
|
||||
turn.saved_spend,
|
||||
turn.savings_estimated_turns,
|
||||
turn.savings_estimated_actual_spend,
|
||||
turn.savings_estimated_saved_spend,
|
||||
)
|
||||
)
|
||||
):
|
||||
raise ValueError("Baseline observation must own an unestimated turn with matching scope and actual cost")
|
||||
return self
|
||||
|
||||
|
||||
class BaselinePublication(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
version: Literal[3] = 3
|
||||
comparison_id: str
|
||||
comparison_started_at: float
|
||||
status: Literal["estimated", "unknown"]
|
||||
reason: str
|
||||
provenance: Literal["observed_identical", "modeled"] | None = None
|
||||
actual_spend: float | None = None
|
||||
baseline_spend: float | None = None
|
||||
input_tokens: int | None = None
|
||||
cache_read_input_tokens: int | None = None
|
||||
cache_creation_5m_input_tokens: int | None = None
|
||||
cache_creation_1h_input_tokens: int | None = None
|
||||
|
||||
@property
|
||||
def costs(self) -> BaselineCosts | None:
|
||||
if self.status != "estimated" or self.actual_spend is None or self.baseline_spend is None:
|
||||
return None
|
||||
return BaselineCosts(self.actual_spend, self.baseline_spend)
|
||||
|
||||
|
||||
def baseline_publication(
|
||||
record: BaselineAccountingRecord, estimate: BaselineEstimate, first_at: float
|
||||
) -> BaselinePublication:
|
||||
costs: Final = price_baseline_comparison(record.pricing, estimate.usage, estimate.provenance)
|
||||
details: Final = estimate.usage.prompt_tokens_details if estimate.usage is not None else None
|
||||
writes: Final = details.cache_creation_token_details if details is not None else None
|
||||
return BaselinePublication(
|
||||
comparison_id=record.scope,
|
||||
comparison_started_at=first_at,
|
||||
status="estimated" if costs is not None else "unknown",
|
||||
reason=estimate.reason if costs is not None or estimate.usage is None else "pricing_unavailable",
|
||||
provenance=estimate.provenance if costs is not None else None,
|
||||
actual_spend=costs.actual if costs is not None else None,
|
||||
baseline_spend=costs.baseline if costs is not None else None,
|
||||
input_tokens=details.text_tokens if details is not None else None,
|
||||
cache_read_input_tokens=details.cached_tokens if details is not None else None,
|
||||
cache_creation_5m_input_tokens=writes.ephemeral_5m_input_tokens if writes is not None else None,
|
||||
cache_creation_1h_input_tokens=writes.ephemeral_1h_input_tokens if writes is not None else None,
|
||||
)
|
||||
|
||||
|
||||
class _Comparison(BaseModel):
|
||||
revision: int
|
||||
published_revision: int
|
||||
initial_equivalent: bool
|
||||
retired: bool
|
||||
history: str | None
|
||||
|
||||
|
||||
class _StoredRecord(BaseModel):
|
||||
data: str
|
||||
publication: str | None
|
||||
conflicted: bool
|
||||
started_at: float
|
||||
|
||||
|
||||
class _Change(BaseModel):
|
||||
request_id: str
|
||||
publication: BaselinePublication
|
||||
api_key: str
|
||||
session_id: str
|
||||
router_name: str
|
||||
baseline_model: str
|
||||
covered_delta: int
|
||||
actual_delta: float
|
||||
savings_delta: float
|
||||
daily: DailyBaselineAttribution | None
|
||||
|
||||
|
||||
class _TransactionManager(Protocol):
|
||||
async def __aenter__(self) -> SupportsRawQueries: ...
|
||||
|
||||
async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
|
||||
|
||||
|
||||
class _TransactionalDatabase(Protocol):
|
||||
def tx(self, *, timeout: timedelta) -> _TransactionManager: ...
|
||||
|
||||
|
||||
_COMPARISONS: Final = TypeAdapter(tuple[_Comparison, ...])
|
||||
_RECORDS: Final = TypeAdapter(tuple[_StoredRecord, ...])
|
||||
_HISTORY: Final = TypeAdapter(BaselineHistory)
|
||||
_PAGE_TIMESTAMPS: Final = 128
|
||||
_TRANSACTION_TIMEOUT: Final = timedelta(seconds=10)
|
||||
|
||||
_CREATE_COMPARISON: Final = """
|
||||
INSERT INTO "LiteLLM_AutoRouterBaselineComparison"
|
||||
(scope, api_key, session_id, router_name, initial_equivalent)
|
||||
VALUES ($1, $2, $3, $4, NOT EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_AutoRouterSession"
|
||||
WHERE api_key = $2 AND session_id = $3 AND router_name = $4
|
||||
)) ON CONFLICT (scope) DO NOTHING
|
||||
"""
|
||||
_LOCK_COMPARISON: Final = """
|
||||
SELECT revision, published_revision, initial_equivalent, retired, history
|
||||
FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope = $1 FOR UPDATE
|
||||
"""
|
||||
_INSERT_RECORD: Final = """
|
||||
INSERT INTO "LiteLLM_AutoRouterBaselineObservation"
|
||||
(request_id, scope, started_at, revision, data)
|
||||
VALUES ($1, $2, $3::float8, $4::bigint, $5)
|
||||
ON CONFLICT (request_id) DO NOTHING
|
||||
"""
|
||||
_MARK_CONFLICT: Final = """
|
||||
UPDATE "LiteLLM_AutoRouterBaselineObservation"
|
||||
SET conflicted = TRUE, revision = $4::bigint
|
||||
WHERE request_id = $1 AND scope = $2 AND data <> $3 AND NOT conflicted
|
||||
"""
|
||||
_READ_PAGE: Final = """
|
||||
WITH times AS (
|
||||
SELECT DISTINCT started_at FROM "LiteLLM_AutoRouterBaselineObservation"
|
||||
WHERE scope = $1 AND revision > $2::bigint
|
||||
AND ($3::float8 IS NULL OR started_at > $3::float8)
|
||||
AND ($5::float8 IS NULL OR (
|
||||
started_at >= $5::float8 AND publication::jsonb->>'status' = 'estimated'
|
||||
))
|
||||
ORDER BY started_at LIMIT $4::int
|
||||
)
|
||||
SELECT data, publication, conflicted, started_at
|
||||
FROM "LiteLLM_AutoRouterBaselineObservation"
|
||||
WHERE scope = $1 AND revision > $2::bigint
|
||||
AND started_at IN (SELECT started_at FROM times)
|
||||
AND ($5::float8 IS NULL OR publication::jsonb->>'status' = 'estimated')
|
||||
ORDER BY started_at, request_id
|
||||
"""
|
||||
_UPDATE_LOGS: Final = """
|
||||
WITH changes AS (
|
||||
SELECT request_id, publication::jsonb AS publication
|
||||
FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb)
|
||||
)
|
||||
UPDATE "LiteLLM_SpendLogs" AS logs
|
||||
SET metadata = (COALESCE(logs.metadata::jsonb, '{}'::jsonb) - 'autorouter_baseline_observation') || jsonb_build_object(
|
||||
'autorouter_savings_estimate', changes.publication,
|
||||
'autorouter_savings', CASE WHEN changes.publication->>'status' = 'estimated' THEN
|
||||
(changes.publication->>'baseline_spend')::float8 - (changes.publication->>'actual_spend')::float8
|
||||
ELSE NULL END
|
||||
)
|
||||
FROM changes WHERE logs.request_id = changes.request_id
|
||||
"""
|
||||
_UPDATE_PUBLICATIONS: Final = """
|
||||
UPDATE "LiteLLM_AutoRouterBaselineObservation" AS observations
|
||||
SET publication = x.publication::text
|
||||
FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb)
|
||||
WHERE observations.request_id = x.request_id
|
||||
"""
|
||||
_UPDATE_SESSIONS: Final = """
|
||||
WITH changes AS (
|
||||
SELECT * FROM jsonb_to_recordset($1::jsonb) AS x(
|
||||
api_key text, session_id text, router_name text, baseline_model text,
|
||||
covered_delta int, actual_delta float8, savings_delta float8
|
||||
)
|
||||
), totals AS (
|
||||
SELECT api_key, session_id, router_name, SUM(covered_delta)::int AS covered_delta,
|
||||
SUM(actual_delta) AS actual_delta, SUM(savings_delta) AS savings_delta
|
||||
FROM changes GROUP BY api_key, session_id, router_name
|
||||
), models AS (
|
||||
SELECT api_key, session_id, router_name, jsonb_object_agg(baseline_model, delta) AS deltas
|
||||
FROM (
|
||||
SELECT api_key, session_id, router_name, baseline_model, SUM(covered_delta)::int AS delta
|
||||
FROM changes GROUP BY api_key, session_id, router_name, baseline_model
|
||||
) grouped GROUP BY api_key, session_id, router_name
|
||||
)
|
||||
UPDATE "LiteLLM_AutoRouterSession" AS session
|
||||
SET saved_spend = session.saved_spend + totals.savings_delta,
|
||||
savings_estimated_turns = session.savings_estimated_turns + totals.covered_delta,
|
||||
savings_estimated_actual_spend = session.savings_estimated_actual_spend + totals.actual_delta,
|
||||
savings_estimated_saved_spend = session.savings_estimated_saved_spend + totals.savings_delta,
|
||||
savings_estimated_baseline_models = (
|
||||
SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) FROM (
|
||||
SELECT key, SUM(value::int)::int AS value FROM (
|
||||
SELECT * FROM jsonb_each_text(session.savings_estimated_baseline_models)
|
||||
UNION ALL SELECT * FROM jsonb_each_text(models.deltas)
|
||||
) combined GROUP BY key HAVING SUM(value::int) > 0
|
||||
) counts
|
||||
)
|
||||
FROM totals JOIN models USING (api_key, session_id, router_name)
|
||||
WHERE session.api_key = totals.api_key AND session.session_id = totals.session_id
|
||||
AND session.router_name = totals.router_name
|
||||
"""
|
||||
|
||||
|
||||
def _primary_transaction(client: PrismaClient) -> _TransactionManager:
|
||||
primary: Final = cast(_TransactionalDatabase, writer_wrapper(client.db))
|
||||
return primary.tx(timeout=_TRANSACTION_TIMEOUT)
|
||||
|
||||
|
||||
def _serialized(model: BaseModel) -> str:
|
||||
return json.dumps(model.model_dump(mode="json"), sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _change(record: BaselineAccountingRecord, old: BaselinePublication | None, new: BaselinePublication) -> _Change:
|
||||
previous: Final = old.costs if old is not None else None
|
||||
current: Final = new.costs
|
||||
return _Change(
|
||||
request_id=record.observation.request_id,
|
||||
publication=new,
|
||||
api_key=record.api_key,
|
||||
session_id=record.session_id,
|
||||
router_name=record.router_name,
|
||||
baseline_model=record.baseline_model,
|
||||
covered_delta=int(current is not None) - int(previous is not None),
|
||||
actual_delta=(current.actual if current is not None else 0.0)
|
||||
- (previous.actual if previous is not None else 0.0),
|
||||
savings_delta=(current.savings if current is not None else 0.0)
|
||||
- (previous.savings if previous is not None else 0.0),
|
||||
daily=record.daily,
|
||||
)
|
||||
|
||||
|
||||
def _project_group(
|
||||
previous: tuple[BaselineHistory, tuple[_Change, ...]], stored: Sequence[_StoredRecord]
|
||||
) -> tuple[BaselineHistory, tuple[_Change, ...]]:
|
||||
history, prior_changes = previous
|
||||
records: Final = tuple(BaselineAccountingRecord.model_validate_json(item.data) for item in stored)
|
||||
observations: Final = tuple(
|
||||
record.observation.model_copy(
|
||||
update=MappingProxyType(
|
||||
{"outcome": "uncertain", "baseline_equivalent": False, "reason": "conflicting_observation"}
|
||||
)
|
||||
)
|
||||
if row.conflicted
|
||||
else record.observation
|
||||
for record, row in zip(records, stored)
|
||||
)
|
||||
advanced, estimates = advance_baseline_history(history, observations)
|
||||
publications: Final = tuple(
|
||||
baseline_publication(
|
||||
record, estimate, advanced.first_at if advanced.first_at is not None else observations[0].started_at
|
||||
)
|
||||
for record, estimate in zip(records, estimates)
|
||||
)
|
||||
changes: Final = tuple(
|
||||
_change(record, old, publication)
|
||||
for record, row, publication in zip(records, stored, publications)
|
||||
for old in (BaselinePublication.model_validate_json(row.publication) if row.publication else None,)
|
||||
if publication != old
|
||||
)
|
||||
return advanced, (*prior_changes, *changes)
|
||||
|
||||
|
||||
async def _publish(db: SupportsRawQueries, changes: Sequence[_Change]) -> None:
|
||||
if not changes:
|
||||
return
|
||||
serialized: Final = json.dumps(tuple(change.model_dump(mode="json") for change in changes), separators=(",", ":"))
|
||||
await db.execute_raw(_UPDATE_LOGS, serialized)
|
||||
await db.execute_raw(_UPDATE_SESSIONS, serialized)
|
||||
for entity, table in DAILY_SPEND_TABLES.items():
|
||||
if adjustments := tuple(
|
||||
change.daily.adjustment(target, change.savings_delta, change.request_id)
|
||||
for change in changes
|
||||
if change.daily is not None and change.savings_delta != 0
|
||||
for target in change.daily.targets
|
||||
if target.entity == entity
|
||||
):
|
||||
statement, values = build_bulk_upsert(table, merge_by_conflict_key(table, adjustments))
|
||||
await db.execute_raw(statement, *values)
|
||||
await db.execute_raw(_UPDATE_PUBLICATIONS, serialized)
|
||||
|
||||
|
||||
class BaselineAccountingStore:
|
||||
def __init__(self, transaction: Callable[[], _TransactionManager]) -> None:
|
||||
self.transaction: Final = transaction
|
||||
|
||||
@classmethod
|
||||
def for_client(cls, client: PrismaClient) -> BaselineAccountingStore:
|
||||
def transaction() -> _TransactionManager:
|
||||
return _primary_transaction(client)
|
||||
|
||||
return cls(transaction)
|
||||
|
||||
async def append(
|
||||
self, record: BaselineAccountingRecord
|
||||
) -> Literal["recorded", "retired", "conflict", "unavailable"]:
|
||||
try:
|
||||
async with self.transaction() as db:
|
||||
await db.execute_raw("SET LOCAL statement_timeout = 5000")
|
||||
await db.execute_raw("SET LOCAL lock_timeout = 1000")
|
||||
await db.execute_raw(
|
||||
_CREATE_COMPARISON, record.scope, record.api_key, record.session_id, record.router_name
|
||||
)
|
||||
rows: Final = _COMPARISONS.validate_python(tuple(await db.query_raw(_LOCK_COMPARISON, record.scope)))
|
||||
if not rows:
|
||||
return "unavailable"
|
||||
revision: Final = rows[0].revision + 1
|
||||
data: Final = _serialized(record)
|
||||
inserted: Final = await db.execute_raw(
|
||||
_INSERT_RECORD,
|
||||
record.observation.request_id,
|
||||
record.scope,
|
||||
record.observation.started_at,
|
||||
revision,
|
||||
data,
|
||||
)
|
||||
if inserted and record.turn is not None:
|
||||
await write_autorouter_turn(db, record.turn)
|
||||
conflicted: Final = (
|
||||
0
|
||||
if inserted
|
||||
else await db.execute_raw(
|
||||
_MARK_CONFLICT, record.observation.request_id, record.scope, data, revision
|
||||
)
|
||||
)
|
||||
canonical: Final = (
|
||||
_RECORDS.validate_python(
|
||||
tuple(
|
||||
await db.query_raw(
|
||||
'SELECT data, publication, conflicted, started_at FROM "LiteLLM_AutoRouterBaselineObservation" '
|
||||
"WHERE request_id=$1 AND scope=$2",
|
||||
record.observation.request_id,
|
||||
record.scope,
|
||||
)
|
||||
)
|
||||
)
|
||||
if not inserted
|
||||
else ()
|
||||
)
|
||||
if not inserted and not canonical:
|
||||
return "conflict"
|
||||
if rows[0].retired:
|
||||
await _publish(
|
||||
db,
|
||||
(
|
||||
_change(
|
||||
BaselineAccountingRecord.model_validate_json(canonical[0].data)
|
||||
if canonical
|
||||
else record,
|
||||
BaselinePublication.model_validate_json(canonical[0].publication)
|
||||
if canonical and canonical[0].publication is not None
|
||||
else None,
|
||||
BaselinePublication(
|
||||
comparison_id=record.scope,
|
||||
comparison_started_at=canonical[0].started_at
|
||||
if canonical
|
||||
else record.observation.started_at,
|
||||
status="unknown",
|
||||
reason="comparison_retired",
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
return "retired"
|
||||
if inserted or conflicted:
|
||||
await self._withdraw(
|
||||
db, record.scope, canonical[0].started_at if canonical else record.observation.started_at
|
||||
)
|
||||
await db.execute_raw(
|
||||
'UPDATE "LiteLLM_AutoRouterBaselineComparison" SET revision = $2::bigint, '
|
||||
"updated_at = CURRENT_TIMESTAMP, attempted_at = NULL WHERE scope = $1",
|
||||
record.scope,
|
||||
revision,
|
||||
)
|
||||
return "recorded"
|
||||
except Exception: # noqa: BLE001 # accounting failure must not change inference or actual billing
|
||||
verbose_proxy_logger.warning("Auto-router baseline observation could not be persisted")
|
||||
return "unavailable"
|
||||
|
||||
async def _pages(
|
||||
self, db: SupportsRawQueries, scope: str, after_revision: int, withdraw_from: float | None = None
|
||||
) -> AsyncIterator[tuple[_StoredRecord, ...]]:
|
||||
cursor: float | None = None
|
||||
while page := _RECORDS.validate_python(
|
||||
tuple(await db.query_raw(_READ_PAGE, scope, after_revision, cursor, _PAGE_TIMESTAMPS, withdraw_from))
|
||||
):
|
||||
yield page
|
||||
cursor = page[-1].started_at # rebind-ok: keyset pagination advances after each complete timestamp group
|
||||
|
||||
async def _withdraw(self, db: SupportsRawQueries, scope: str, started_at: float) -> None:
|
||||
async for page in self._pages(db, scope, 0, withdraw_from=started_at):
|
||||
await _publish(
|
||||
db,
|
||||
tuple(
|
||||
_change(
|
||||
BaselineAccountingRecord.model_validate_json(row.data),
|
||||
previous,
|
||||
BaselinePublication(
|
||||
comparison_id=scope,
|
||||
comparison_started_at=min(previous.comparison_started_at, started_at),
|
||||
status="unknown",
|
||||
reason="pending_projection",
|
||||
),
|
||||
)
|
||||
for row in page
|
||||
if row.publication is not None
|
||||
for previous in (BaselinePublication.model_validate_json(row.publication),)
|
||||
),
|
||||
)
|
||||
|
||||
async def retire_before(self, cutoff: datetime, batch_size: int, timeout_ms: int) -> None:
|
||||
async with self.transaction() as db:
|
||||
await db.execute_raw(f"SET LOCAL statement_timeout = {max(1, timeout_ms)}")
|
||||
await db.execute_raw(f"SET LOCAL lock_timeout = {max(1, timeout_ms)}")
|
||||
await db.execute_raw(
|
||||
'WITH expired AS (SELECT scope FROM "LiteLLM_AutoRouterBaselineComparison" '
|
||||
"WHERE NOT retired AND updated_at < $1::timestamptz ORDER BY updated_at "
|
||||
"LIMIT $2::int FOR UPDATE SKIP LOCKED) "
|
||||
'UPDATE "LiteLLM_AutoRouterBaselineComparison" AS comparison '
|
||||
"SET retired=TRUE, history=NULL FROM expired WHERE comparison.scope=expired.scope",
|
||||
cutoff,
|
||||
batch_size,
|
||||
)
|
||||
await db.execute_raw(
|
||||
'DELETE FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id IN ('
|
||||
'SELECT event.request_id FROM "LiteLLM_AutoRouterBaselineObservation" AS event '
|
||||
'JOIN "LiteLLM_AutoRouterBaselineComparison" AS comparison USING (scope) '
|
||||
"WHERE comparison.retired AND comparison.updated_at < $1::timestamptz "
|
||||
"LIMIT $2::int)",
|
||||
cutoff,
|
||||
batch_size,
|
||||
)
|
||||
|
||||
async def project(self, scope: str) -> Literal["published", "unchanged", "unavailable"]:
|
||||
try:
|
||||
async with self.transaction() as db:
|
||||
await db.execute_raw("SET LOCAL statement_timeout = 5000")
|
||||
await db.execute_raw("SET LOCAL lock_timeout = 1000")
|
||||
rows: Final = _COMPARISONS.validate_python(tuple(await db.query_raw(_LOCK_COMPARISON, scope)))
|
||||
if not rows or rows[0].retired or rows[0].revision == rows[0].published_revision:
|
||||
return "unchanged"
|
||||
missing_log: Final = await db.query_raw(
|
||||
'SELECT 1 FROM "LiteLLM_AutoRouterBaselineObservation" AS observation '
|
||||
'WHERE scope=$1 AND publication IS NULL AND NOT EXISTS (SELECT 1 FROM "LiteLLM_SpendLogs" AS log '
|
||||
"WHERE log.request_id=observation.request_id) LIMIT 1",
|
||||
scope,
|
||||
)
|
||||
if missing_log:
|
||||
return "unavailable"
|
||||
state: Final = rows[0]
|
||||
checkpoint: Final = (
|
||||
_HISTORY.validate_json(state.history)
|
||||
if state.history is not None
|
||||
else BaselineHistory(equivalent=state.initial_equivalent)
|
||||
)
|
||||
changed: Final = await db.query_raw(
|
||||
'SELECT 1 FROM "LiteLLM_AutoRouterBaselineObservation" '
|
||||
"WHERE scope = $1 AND revision > $2::bigint AND started_at <= $3::float8 LIMIT 1",
|
||||
scope,
|
||||
state.published_revision,
|
||||
checkpoint.last_at,
|
||||
)
|
||||
history = BaselineHistory(equivalent=state.initial_equivalent) if changed else checkpoint
|
||||
async for page in self._pages(db, scope, 0 if changed else state.published_revision):
|
||||
history, updates = reduce(
|
||||
_project_group,
|
||||
(tuple(group) for _, group in groupby(page, key=lambda item: item.started_at)),
|
||||
(history, ()),
|
||||
)
|
||||
await _publish(db, updates)
|
||||
await db.execute_raw(
|
||||
'UPDATE "LiteLLM_AutoRouterBaselineComparison" '
|
||||
"SET published_revision = revision, history = $2 WHERE scope = $1",
|
||||
scope,
|
||||
_HISTORY.dump_json(history).decode(),
|
||||
)
|
||||
return "published"
|
||||
except Exception: # noqa: BLE001 # rollback leaves the durable revision dirty for a later flush
|
||||
verbose_proxy_logger.warning("Auto-router baseline projection remains pending")
|
||||
return "unavailable"
|
||||
|
||||
|
||||
class _Scope(BaseModel):
|
||||
scope: str
|
||||
|
||||
|
||||
_SCOPES: Final = TypeAdapter(tuple[_Scope, ...])
|
||||
_CLAIM_DIRTY: Final = """
|
||||
WITH candidates AS (
|
||||
SELECT scope FROM "LiteLLM_AutoRouterBaselineComparison"
|
||||
WHERE NOT retired AND revision <> published_revision
|
||||
AND (attempted_at IS NULL OR attempted_at < CURRENT_TIMESTAMP - INTERVAL '30 seconds')
|
||||
ORDER BY attempted_at NULLS FIRST, updated_at, scope LIMIT 32 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE "LiteLLM_AutoRouterBaselineComparison" AS comparison
|
||||
SET attempted_at = CURRENT_TIMESTAMP FROM candidates
|
||||
WHERE comparison.scope = candidates.scope RETURNING comparison.scope
|
||||
"""
|
||||
|
||||
|
||||
async def _flush_records(
|
||||
store: BaselineAccountingStore, records: Sequence[BaselineAccountingRecord]
|
||||
) -> tuple[BaselineAccountingRecord, ...]:
|
||||
slots: Final = asyncio.Semaphore(4)
|
||||
|
||||
async def append(record: BaselineAccountingRecord) -> bool:
|
||||
async with slots:
|
||||
return await store.append(record) == "unavailable"
|
||||
|
||||
failed: Final = await asyncio.gather(*(append(record) for record in records))
|
||||
return tuple(record for record, retry in zip(records, failed) if retry)
|
||||
|
||||
|
||||
async def flush_baseline_accounting(client: PrismaClient) -> None:
|
||||
from litellm.proxy.utils import request_spend_log_flush
|
||||
|
||||
store: Final = BaselineAccountingStore.for_client(client)
|
||||
async with client.baseline_accounting_lock:
|
||||
batch: Final = tuple(client.baseline_accounting_transactions[:32])
|
||||
client.baseline_accounting_transactions = client.baseline_accounting_transactions[
|
||||
32:
|
||||
] # rebind-ok: drain under lock
|
||||
more_queued: Final = bool(client.baseline_accounting_transactions)
|
||||
try:
|
||||
remaining: Final = await asyncio.wait_for(_flush_records(store, batch), timeout=5)
|
||||
except (Exception, asyncio.CancelledError) as error: # noqa: BLE001 # unknown acknowledgements can be replayed safely
|
||||
async with client.baseline_accounting_lock:
|
||||
client.baseline_accounting_transactions.extend(batch)
|
||||
if isinstance(error, asyncio.CancelledError):
|
||||
raise
|
||||
return
|
||||
async with client.baseline_accounting_lock:
|
||||
client.baseline_accounting_transactions.extend(remaining)
|
||||
if more_queued and len(remaining) < len(batch):
|
||||
request_spend_log_flush(client)
|
||||
try:
|
||||
async with store.transaction() as db:
|
||||
await db.execute_raw("SET LOCAL statement_timeout = 1000")
|
||||
scopes: Final = _SCOPES.validate_python(tuple(await db.query_raw(_CLAIM_DIRTY)))
|
||||
slots: Final = asyncio.Semaphore(4)
|
||||
|
||||
async def project(item: _Scope) -> str:
|
||||
async with slots:
|
||||
return await store.project(item.scope)
|
||||
|
||||
outcomes: Final = await asyncio.wait_for(asyncio.gather(*(project(item) for item in scopes)), timeout=5)
|
||||
if len(scopes) == 32 and "published" in outcomes:
|
||||
request_spend_log_flush(client)
|
||||
except Exception: # noqa: BLE001 # durable dirty comparisons remain eligible after the retry interval
|
||||
verbose_proxy_logger.warning("Auto-router baseline projection will retry on a later spend flush")
|
||||
|
|
@ -14,6 +14,8 @@ from itertools import groupby
|
|||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
DailySpendEntity = Literal["user", "team", "org", "tag", "end_user", "agent"]
|
||||
|
||||
SqlValue = str | int | float | None
|
||||
|
|
@ -43,6 +45,36 @@ DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingP
|
|||
}
|
||||
)
|
||||
|
||||
_ENTITY_INPUT_KEYS: Final[Mapping[DailySpendEntity, str]] = MappingProxyType(
|
||||
{
|
||||
"user": "user",
|
||||
"team": "team_id",
|
||||
"org": "organization_id",
|
||||
"end_user": "end_user",
|
||||
"agent": "agent_id",
|
||||
"tag": "request_tags",
|
||||
}
|
||||
)
|
||||
_TAGS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEntity) -> tuple[str | None, ...]:
|
||||
key: Final = _ENTITY_INPUT_KEYS[entity]
|
||||
if key not in payload:
|
||||
return ()
|
||||
value: Final = payload[key]
|
||||
if entity == "tag":
|
||||
if value is None:
|
||||
return ()
|
||||
tags: Final = _TAGS.validate_json(value) if isinstance(value, str) else _TAGS.validate_python(value)
|
||||
return tuple(dict.fromkeys(tags))
|
||||
if value is None:
|
||||
return (None,) if entity == "user" else ()
|
||||
if not isinstance(value, str) or (entity == "end_user" and not value):
|
||||
return ()
|
||||
return (value,)
|
||||
|
||||
|
||||
# The unique constraint's columns after the entity id, in constraint order. A NULL can
|
||||
# never match itself in a unique index, so every one of these is normalized to '': the
|
||||
# conflict target has to be NULL-free or the row is re-inserted on every single flush.
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, overload
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import LiteralString, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -51,6 +52,7 @@ from litellm.proxy.common_utils.user_api_key_cache import project_cache_key
|
|||
from litellm.proxy.db.daily_spend_bulk_upsert import (
|
||||
DAILY_SPEND_TABLES,
|
||||
build_bulk_upsert,
|
||||
daily_spend_entity_ids,
|
||||
merge_by_conflict_key,
|
||||
)
|
||||
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
|
||||
|
|
@ -82,6 +84,8 @@ from litellm.repositories.prisma_protocols import BatchTable
|
|||
from litellm.types.utils import CallTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
|
||||
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
else:
|
||||
PrismaClient = Any
|
||||
|
|
@ -89,6 +93,7 @@ else:
|
|||
|
||||
|
||||
RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value})
|
||||
_SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _org_member_transaction_key(org_id: str, user_id: str) -> str:
|
||||
|
|
@ -579,25 +584,31 @@ class DBSpendUpdateWriter:
|
|||
metadata_raw: Final = payload.get("metadata")
|
||||
if not metadata_raw:
|
||||
return
|
||||
metadata: Final = json.loads(metadata_raw)
|
||||
if not isinstance(metadata, dict) or not metadata.get("routing_decision"):
|
||||
metadata: Final = _SPEND_METADATA_ADAPTER.validate_json(metadata_raw)
|
||||
routing_decision: Final = metadata.get("routing_decision")
|
||||
if not isinstance(routing_decision, Mapping) or not routing_decision:
|
||||
return
|
||||
from litellm.proxy.db.autorouter_session_rollup import (
|
||||
build_autorouter_turn_transaction,
|
||||
)
|
||||
|
||||
usage_object_raw: Final = metadata.get("usage_object")
|
||||
cost_breakdown: Final = metadata.get("cost_breakdown")
|
||||
savings_estimate: Final = metadata.get("autorouter_savings_estimate")
|
||||
savings_spend: Final = compute_savings_spend(
|
||||
model=payload.get("model"),
|
||||
custom_llm_provider=payload.get("custom_llm_provider"),
|
||||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")),
|
||||
routing_decision=metadata.get("routing_decision"),
|
||||
routing_decision=routing_decision,
|
||||
usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
|
||||
model_id=payload.get("model_id"),
|
||||
llm_router=get_llm_router,
|
||||
cost_breakdown=metadata.get("cost_breakdown"),
|
||||
cost_breakdown=cost_breakdown if isinstance(cost_breakdown, Mapping) else None,
|
||||
recorded_autorouter_savings=metadata.get("autorouter_savings"),
|
||||
recorded_autorouter_savings_estimate=(
|
||||
savings_estimate if isinstance(savings_estimate, Mapping) else None
|
||||
),
|
||||
billed_at=payload.get("endTime"),
|
||||
)
|
||||
transaction: Final = build_autorouter_turn_transaction(
|
||||
|
|
@ -605,6 +616,11 @@ class DBSpendUpdateWriter:
|
|||
metadata=metadata,
|
||||
saved_spend=savings_spend.autorouter,
|
||||
)
|
||||
try:
|
||||
if await self._enqueue_baseline_accounting(payload, metadata, transaction, prisma_client):
|
||||
return
|
||||
except Exception: # noqa: BLE001 # optional baseline capture must preserve the original actual-spend rollup
|
||||
verbose_proxy_logger.warning("Auto-router baseline observation was unavailable; actual turn retained")
|
||||
if transaction is None:
|
||||
return
|
||||
async with prisma_client._autorouter_turn_transactions_lock:
|
||||
|
|
@ -612,6 +628,95 @@ class DBSpendUpdateWriter:
|
|||
except Exception as e: # noqa: BLE001 # a metrics enqueue must never fail the spend write
|
||||
verbose_proxy_logger.debug("_enqueue_autorouter_turn_transaction error (non-blocking): %s", e)
|
||||
|
||||
async def _enqueue_baseline_accounting(
|
||||
self,
|
||||
payload: SpendLogsPayload,
|
||||
metadata: Mapping[str, object],
|
||||
turn: "AutoRouterTurnTransaction | None",
|
||||
prisma_client: "PrismaClient",
|
||||
) -> bool:
|
||||
from litellm.proxy.db.baseline_accounting import (
|
||||
BaselineAccountingRecord,
|
||||
)
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation
|
||||
from litellm.proxy.spend_tracking.savings import baseline_cost_snapshot
|
||||
|
||||
serialized: Final = metadata.get("autorouter_baseline_observation")
|
||||
if not isinstance(serialized, str):
|
||||
return False
|
||||
captured: Final = CapturedBaselineObservation.model_validate_json(serialized)
|
||||
if captured.api_key != payload["api_key"] or captured.session_id != payload["session_id"]:
|
||||
return False
|
||||
decision: Final = _SPEND_METADATA_ADAPTER.validate_python(
|
||||
metadata.get("routing_decision") or MappingProxyType({})
|
||||
)
|
||||
breakdown: Final = _SPEND_METADATA_ADAPTER.validate_python(
|
||||
metadata.get("cost_breakdown") or MappingProxyType({})
|
||||
)
|
||||
daily: Final = await self._baseline_daily_attribution(payload, prisma_client)
|
||||
record: Final = BaselineAccountingRecord(
|
||||
scope=captured.scope,
|
||||
api_key=captured.api_key,
|
||||
session_id=captured.session_id,
|
||||
router_name=captured.router_name,
|
||||
baseline_model=captured.baseline_model,
|
||||
observation=captured.observation.model_copy(update=MappingProxyType({"request_id": payload["request_id"]})),
|
||||
pricing=baseline_cost_snapshot(captured.model, captured.prices, payload["spend"], breakdown, decision),
|
||||
turn=turn,
|
||||
daily=daily,
|
||||
)
|
||||
async with prisma_client.baseline_accounting_lock:
|
||||
if len(prisma_client.baseline_accounting_transactions) >= 10000:
|
||||
verbose_proxy_logger.warning("Auto-router baseline observation queue is full")
|
||||
return False
|
||||
prisma_client.baseline_accounting_transactions.append(record)
|
||||
from litellm.proxy.utils import request_spend_log_flush
|
||||
|
||||
request_spend_log_flush(prisma_client)
|
||||
return True
|
||||
|
||||
async def _baseline_daily_attribution(
|
||||
self,
|
||||
payload: SpendLogsPayload,
|
||||
prisma_client: "PrismaClient",
|
||||
) -> "DailyBaselineAttribution | None":
|
||||
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution, DailyBaselineTarget
|
||||
|
||||
normalized: Final = cast(SpendLogsPayload, MappingProxyType({**payload, "end_user_id": payload["end_user"]}))
|
||||
bases: Final = tuple(
|
||||
zip(
|
||||
DAILY_SPEND_TABLES,
|
||||
await asyncio.gather(
|
||||
*(
|
||||
self._common_add_spend_log_transaction_to_daily_transaction( # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a validated spend payload
|
||||
normalized,
|
||||
prisma_client,
|
||||
"request_tags" if entity == "tag" else entity,
|
||||
)
|
||||
for entity in DAILY_SPEND_TABLES
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
base: Final = next((base for _, base in bases if base is not None), None)
|
||||
if base is None:
|
||||
return None
|
||||
return DailyBaselineAttribution(
|
||||
date=base["date"],
|
||||
api_key=base["api_key"],
|
||||
model=base.get("model"),
|
||||
custom_llm_provider=base.get("custom_llm_provider"),
|
||||
model_group=base.get("model_group"),
|
||||
endpoint=base.get("endpoint"),
|
||||
mcp_namespaced_tool_name=base.get("mcp_namespaced_tool_name"),
|
||||
targets=tuple(
|
||||
DailyBaselineTarget(entity=entity, entity_id=identity)
|
||||
for entity, values in bases
|
||||
if values is not None
|
||||
for identity in daily_spend_entity_ids(payload, entity)
|
||||
),
|
||||
)
|
||||
|
||||
def _enqueue_tool_registry_upsert(
|
||||
self,
|
||||
kwargs: dict | None,
|
||||
|
|
@ -2322,21 +2427,13 @@ class DBSpendUpdateWriter:
|
|||
prisma_client: PrismaClient,
|
||||
type: Literal["user", "team", "org", "request_tags", "end_user", "agent"] = "user",
|
||||
) -> BaseDailySpendTransaction | None:
|
||||
common_expected_keys: Final = ["startTime", "api_key"]
|
||||
if type == "user":
|
||||
expected_keys = ["user", *common_expected_keys]
|
||||
elif type == "team":
|
||||
expected_keys = ["team_id", *common_expected_keys]
|
||||
elif type == "org":
|
||||
expected_keys = ["organization_id", *common_expected_keys]
|
||||
elif type == "request_tags":
|
||||
expected_keys = ["request_tags", *common_expected_keys]
|
||||
elif type == "end_user":
|
||||
expected_keys = ["end_user_id", *common_expected_keys]
|
||||
elif type == "agent":
|
||||
expected_keys = ["agent_id", *common_expected_keys]
|
||||
else:
|
||||
raise ValueError(f"Invalid type: {type}")
|
||||
entity: Final = "tag" if type == "request_tags" else type
|
||||
identity_payload: Final = (
|
||||
MappingProxyType({**payload, "end_user": payload.get("end_user_id")}) if type == "end_user" else payload
|
||||
)
|
||||
if not daily_spend_entity_ids(identity_payload, entity):
|
||||
return None
|
||||
expected_keys: Final = ("startTime", "api_key")
|
||||
if not all(key in payload for key in expected_keys):
|
||||
verbose_proxy_logger.debug(
|
||||
"Missing expected keys: %s, in payload, skipping from daily_user_spend_transactions", expected_keys
|
||||
|
|
@ -2399,6 +2496,7 @@ class DBSpendUpdateWriter:
|
|||
usage_object=usage_obj,
|
||||
cost_breakdown=_metadata.get("cost_breakdown"),
|
||||
recorded_autorouter_savings=_metadata.get("autorouter_savings"),
|
||||
recorded_autorouter_savings_estimate=_metadata.get("autorouter_savings_estimate"),
|
||||
billed_at=payload.get("endTime"),
|
||||
)
|
||||
timed_duration_ms: Final = _timed_request_duration_ms(payload, request_status, is_internal_call)
|
||||
|
|
@ -2597,14 +2695,10 @@ class DBSpendUpdateWriter:
|
|||
verbose_proxy_logger.debug("request_tags is None for request. Skipping incrementing tag spend.")
|
||||
return
|
||||
|
||||
request_tags: Sequence[str] = []
|
||||
if isinstance(payload["request_tags"], str):
|
||||
request_tags = json.loads(payload["request_tags"])
|
||||
elif isinstance(payload["request_tags"], list):
|
||||
request_tags = payload["request_tags"]
|
||||
else:
|
||||
raise ValueError(f"Invalid request_tags: {payload['request_tags']}")
|
||||
request_tags: Final = daily_spend_entity_ids(payload, "tag")
|
||||
for tag in request_tags:
|
||||
if tag is None:
|
||||
continue
|
||||
endpoint_str = base_daily_transaction.get("endpoint") or ""
|
||||
daily_transaction_key = f"{tag}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}"
|
||||
daily_transaction = DailyTagSpendTransaction(
|
||||
|
|
|
|||
|
|
@ -549,6 +549,17 @@ class SpendLogCleanup:
|
|||
Prune auto-router session rollup rows, which carry their own retention horizon.
|
||||
"""
|
||||
session_cutoff: Final = datetime.now(timezone.utc) - timedelta(seconds=float(retention_seconds))
|
||||
from litellm.proxy.db.baseline_accounting import BaselineAccountingStore
|
||||
|
||||
if remaining_ms := self._remaining_timeout_ms(deadline)():
|
||||
try:
|
||||
await BaselineAccountingStore.for_client(prisma_client).retire_before(
|
||||
session_cutoff,
|
||||
self.batch_size,
|
||||
remaining_ms,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job
|
||||
verbose_proxy_logger.warning("Auto-router baseline retention remains pending")
|
||||
sessions_result: Final = await self._delete_old_autorouter_session_rows(prisma_client, session_cutoff, deadline)
|
||||
verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted)
|
||||
return (sessions_result,)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import os
|
|||
from typing import Final, Literal
|
||||
|
||||
from . import *
|
||||
from .autorouter_baseline_cache import AutoRouterBaselineCache
|
||||
from .cache_control_check import _PROXY_CacheControlCheck
|
||||
from .litellm_skills import SkillsInjectionHook
|
||||
from .max_budget_per_session_limiter import _PROXY_MaxBudgetPerSessionHandler
|
||||
|
|
@ -25,6 +26,7 @@ PROXY_HOOKS: Final = {
|
|||
"max_budget_per_session_limiter": _PROXY_MaxBudgetPerSessionHandler,
|
||||
"sensitive_data_routing": _PROXY_SensitiveDataRoutingHandler,
|
||||
"prompt_cache_prediction": PromptCacheObserver,
|
||||
"autorouter_baseline_cache": AutoRouterBaselineCache,
|
||||
}
|
||||
|
||||
## FEATURE FLAG HOOKS ##
|
||||
|
|
|
|||
344
litellm/proxy/hooks/autorouter_baseline_cache.py
Normal file
344
litellm/proxy/hooks/autorouter_baseline_cache.py
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_litellm_metadata_from_kwargs, # pyright: ignore[reportUnknownVariableType] # legacy metadata boundary validated below
|
||||
)
|
||||
from litellm.llms.anthropic.prompt_cache_prediction import (
|
||||
CountedPromptCachePlan,
|
||||
NativePredictionTarget,
|
||||
TokenCounter,
|
||||
UnsupportedCachePlan,
|
||||
UnsupportedPredictionTarget,
|
||||
count_cache_plan,
|
||||
count_prompt_tokens,
|
||||
parse_cache_plan,
|
||||
resolve_baseline_prediction_target,
|
||||
supported_baseline_recipient,
|
||||
supported_prediction_headers,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation
|
||||
from litellm.proxy.spend_tracking.savings import (
|
||||
_effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner
|
||||
_proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner
|
||||
)
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
from litellm.types.utils import CallTypes, ModelInfo, Usage
|
||||
from litellm.utils import get_prompt_cache_min_tokens
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.router import Router
|
||||
|
||||
_METADATA: Final = TypeAdapter(Mapping[str, object])
|
||||
_PRICES: Final[TypeAdapter[ModelInfo | None]] = TypeAdapter(ModelInfo | None)
|
||||
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_COUNT_TIMEOUT: Final = 3.0
|
||||
_MAX_COUNTS: Final = 4096
|
||||
|
||||
|
||||
class CapturedBaselineObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
scope: str
|
||||
api_key: str
|
||||
session_id: str
|
||||
router_name: str
|
||||
baseline_model: str
|
||||
model: str
|
||||
prices: ModelInfo | None
|
||||
observation: BaselineObservation
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BaselineCacheContext:
|
||||
collector: AutoRouterBaselineCache
|
||||
capture: CapturedBaselineObservation
|
||||
target: NativePredictionTarget | UnsupportedPredictionTarget
|
||||
baseline_deployment_id: str
|
||||
invalidated: str | None = None
|
||||
|
||||
|
||||
class _Metadata(BaseModel):
|
||||
model_config = ConfigDict(strict=True, arbitrary_types_allowed=True)
|
||||
route: BaselineRouteStamp = Field(alias="_autorouter_baseline_route")
|
||||
user_api_key_hash: str = Field(min_length=1)
|
||||
session_id: str | None = None
|
||||
|
||||
|
||||
class _WireEvent(BaseModel):
|
||||
model_config = ConfigDict(strict=True, arbitrary_types_allowed=True)
|
||||
httpx_response: httpx.Response
|
||||
api_call_start_time: datetime
|
||||
completion_start_time: datetime
|
||||
custom_llm_provider: str
|
||||
stream: bool = False
|
||||
prompt_cache_response_complete: bool = False
|
||||
|
||||
|
||||
class _ResponseUsage(BaseModel):
|
||||
model_config = ConfigDict(strict=True, from_attributes=True)
|
||||
usage: Usage | None = None
|
||||
|
||||
|
||||
def _digest(value: object) -> str:
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
class AutoRouterBaselineCache(CustomLogger):
|
||||
def __init__(
|
||||
self,
|
||||
prisma_client: PrismaClient | None,
|
||||
router: Callable[[], Router | None] = _proxy_llm_router,
|
||||
token_counter: TokenCounter | None = None,
|
||||
clock: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
super().__init__() # pyright: ignore[reportUnknownMemberType] # legacy callback constructor
|
||||
self.router: Final = router
|
||||
self.token_counter: Final = token_counter
|
||||
self.clock: Final = clock
|
||||
self.count_slots: Final = asyncio.Semaphore(8)
|
||||
self.counts: Mapping[str, tuple[int, float]] = MappingProxyType({})
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: CallTypes | None) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if not isinstance(logging_obj, Logging) or call_type != CallTypes.anthropic_messages:
|
||||
return
|
||||
try:
|
||||
metadata: Final = _METADATA.validate_python(
|
||||
get_litellm_metadata_from_kwargs(
|
||||
{"litellm_params": kwargs} # mutable-ok: legacy metadata owner requires a dictionary
|
||||
)
|
||||
)
|
||||
if metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY):
|
||||
return
|
||||
if logging_obj.baseline_cache_context is not None:
|
||||
await invalidate_baseline_cache(logging_obj, "retried_request")
|
||||
return
|
||||
request: Final = _Metadata.model_validate(metadata)
|
||||
session: Final = kwargs.get("litellm_session_id") or request.session_id or logging_obj.litellm_session_id
|
||||
if not isinstance(session, str) or not session or len(session) > 256:
|
||||
return
|
||||
router: Final = self.router()
|
||||
deployment: Final = router.get_deployment(request.route.baseline_deployment_id) if router else None
|
||||
if deployment is None:
|
||||
return
|
||||
target: Final = resolve_baseline_prediction_target(deployment.litellm_params)
|
||||
prices: Final = _PRICES.validate_python(
|
||||
_effective_model_info(router, request.route.baseline_deployment_id, request.route.baseline_model)
|
||||
)
|
||||
scope: Final = "autorouter-baseline:v3:" + _digest(
|
||||
(
|
||||
request.user_api_key_hash,
|
||||
session,
|
||||
request.route.router_name,
|
||||
request.route.baseline_deployment_id,
|
||||
deployment.litellm_params.model_dump(mode="json"),
|
||||
prices,
|
||||
)
|
||||
)
|
||||
started: Final = logging_obj.start_time.timestamp()
|
||||
capture: Final = CapturedBaselineObservation(
|
||||
scope=scope,
|
||||
api_key=request.user_api_key_hash,
|
||||
session_id=session,
|
||||
router_name=request.route.router_name,
|
||||
baseline_model=request.route.baseline_model,
|
||||
model=target.model if isinstance(target, NativePredictionTarget) else request.route.baseline_model,
|
||||
prices=prices,
|
||||
observation=BaselineObservation(
|
||||
request_id=logging_obj.litellm_call_id,
|
||||
started_at=started,
|
||||
available_at=started,
|
||||
outcome="uncertain",
|
||||
baseline_equivalent=False,
|
||||
reason="incomplete_response",
|
||||
),
|
||||
)
|
||||
logging_obj.baseline_cache_context = BaselineCacheContext(
|
||||
self, capture, target, request.route.baseline_deployment_id
|
||||
)
|
||||
except Exception: # noqa: BLE001 # optional observation cannot fail inference
|
||||
verbose_proxy_logger.warning("Auto-router baseline observation could not be initialized")
|
||||
|
||||
async def _count(self, target: NativePredictionTarget, body: Mapping[str, JsonValue]) -> int | None:
|
||||
key: Final = _digest((target.model, target.api_key, target.api_base, _JSON_BODY.validate_python(body)))
|
||||
now: Final = self.clock()
|
||||
cached: Final = self.counts.get(key)
|
||||
if cached is not None and cached[1] > now:
|
||||
return cached[0]
|
||||
async with self.count_slots:
|
||||
tokens: Final = (
|
||||
await self.token_counter(target.model, target.api_key, body)
|
||||
if self.token_counter is not None
|
||||
else await count_prompt_tokens(target.model, target.api_key, body, api_base=target.api_base)
|
||||
)
|
||||
if tokens is None or tokens < 0:
|
||||
return None
|
||||
retained: Final = tuple((k, v) for k, v in self.counts.items() if v[1] > now and k != key)[-(_MAX_COUNTS - 1) :]
|
||||
self.counts = MappingProxyType(dict((*retained, (key, (tokens, now + 3600)))))
|
||||
return tokens
|
||||
|
||||
async def plan(
|
||||
self, target: NativePredictionTarget, wire: httpx.Request, body: Mapping[str, JsonValue], usage: Usage | None
|
||||
) -> tuple[CountedPromptCachePlan | None, str | None]:
|
||||
if not supported_prediction_headers(wire.headers):
|
||||
return None, "unsupported_request_headers"
|
||||
plan: Final = parse_cache_plan(body)
|
||||
if isinstance(plan, UnsupportedCachePlan):
|
||||
return None, plan.reason
|
||||
details: Final = usage.prompt_tokens_details if usage is not None else None
|
||||
if (
|
||||
not plan.breakpoints
|
||||
and details is not None
|
||||
and ((details.cached_tokens or 0) + (details.cache_creation_tokens or 0))
|
||||
):
|
||||
return None, "implicit_cache_without_breakpoints"
|
||||
|
||||
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
|
||||
return await self._count(target, body)
|
||||
|
||||
try:
|
||||
counted: Final = await asyncio.wait_for(
|
||||
count_cache_plan(target.model, target.api_key, plan, token_counter=count), timeout=_COUNT_TIMEOUT
|
||||
)
|
||||
return (None, counted.reason) if isinstance(counted, UnsupportedCachePlan) else (counted, None)
|
||||
except TimeoutError:
|
||||
return None, "token_count_timeout"
|
||||
except Exception: # noqa: BLE001 # token counting cannot fail a completed request
|
||||
return None, "token_count_unavailable"
|
||||
|
||||
|
||||
async def invalidate_baseline_cache(logging_obj: Logging, reason: str, *, completed: bool = False) -> None:
|
||||
context: Final = logging_obj.baseline_cache_context
|
||||
if context is not None:
|
||||
logging_obj.baseline_cache_context = replace(
|
||||
context, invalidated=reason
|
||||
) # rebind-ok: request-owned retry marker
|
||||
logging_obj.baseline_observation = context.capture.model_copy(
|
||||
update=MappingProxyType(
|
||||
{ # rebind-ok: capture uncertainty for failure logging
|
||||
"observation": context.capture.observation.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"available_at": max(context.capture.observation.started_at, context.collector.clock()),
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def finalize_baseline_cache(logging_obj: Logging, response_obj: object) -> None:
|
||||
context: Final = logging_obj.baseline_cache_context
|
||||
if context is None:
|
||||
return
|
||||
try:
|
||||
capture: Final = await _capture(context, logging_obj, response_obj)
|
||||
if logging_obj.baseline_cache_context is context:
|
||||
logging_obj.baseline_observation = capture # rebind-ok: attach only to the captured request owner
|
||||
except Exception: # noqa: BLE001 # observation failures must preserve inference and billing
|
||||
await invalidate_baseline_cache(logging_obj, "observation_unavailable")
|
||||
|
||||
|
||||
async def _capture(
|
||||
context: BaselineCacheContext, logging_obj: Logging, response_obj: object
|
||||
) -> CapturedBaselineObservation:
|
||||
original: Final = context.capture.observation
|
||||
details: Final = _METADATA.validate_python(logging_obj.model_call_details)
|
||||
if details.get("cache_hit") is True:
|
||||
return context.capture.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"observation": original.model_copy(
|
||||
update=MappingProxyType({"outcome": "response_cache", "reason": "response_cache_hit"})
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
event: Final = _WireEvent.model_validate(details)
|
||||
wire: Final = event.httpx_response.request
|
||||
usage: Final = _ResponseUsage.model_validate(response_obj).usage
|
||||
complete: Final = (
|
||||
event.custom_llm_provider == "anthropic"
|
||||
and event.httpx_response.status_code == 200
|
||||
and (not event.stream or event.prompt_cache_response_complete)
|
||||
)
|
||||
started: Final = original.started_at
|
||||
available: Final = event.completion_start_time.timestamp()
|
||||
if context.invalidated or not complete or not started <= available <= context.collector.clock():
|
||||
return context.capture.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"observation": original.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"available_at": max(started, context.collector.clock()),
|
||||
"reason": context.invalidated or "incomplete_response",
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
target: Final = context.target
|
||||
if isinstance(target, UnsupportedPredictionTarget) or not supported_baseline_recipient(target, wire):
|
||||
return context.capture.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"observation": original.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"available_at": available,
|
||||
"reason": target.reason
|
||||
if isinstance(target, UnsupportedPredictionTarget)
|
||||
else "unsupported_baseline_recipient",
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
body: Final = _JSON_BODY.validate_json(wire.content)
|
||||
same: Final = (
|
||||
logging_obj.get_router_model_id() == context.baseline_deployment_id and body.get("model") == target.model
|
||||
)
|
||||
plan, reason = await context.collector.plan(target, wire, body, usage)
|
||||
minimum: Final = get_prompt_cache_min_tokens(target.model)
|
||||
return context.capture.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"observation": BaselineObservation(
|
||||
request_id=original.request_id,
|
||||
started_at=started,
|
||||
available_at=available,
|
||||
outcome="complete",
|
||||
baseline_equivalent=same,
|
||||
usage=usage,
|
||||
plan=plan,
|
||||
minimum_cache_tokens=minimum,
|
||||
reason=reason,
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
|
@ -556,6 +556,9 @@ class _SessionAggRow(BaseModel):
|
|||
total_tokens: int
|
||||
spend: float
|
||||
saved_spend: float
|
||||
savings_estimated_turns: int = 0
|
||||
savings_estimated_actual_spend: float = 0.0
|
||||
savings_estimated_saved_spend: float = 0.0
|
||||
classifier_cost: float
|
||||
classifier_cost_recorded_turns: int
|
||||
session_seconds: float
|
||||
|
|
@ -582,9 +585,19 @@ def _cache_bucket(turns: int, hits: int) -> AutoRouterCacheBucket:
|
|||
return AutoRouterCacheBucket(turns=turns, hits=hits, hit_rate_pct=_pct(hits, turns))
|
||||
|
||||
|
||||
def _savings_cohort(
|
||||
turns: int, estimated_turns: int, actual_spend: float, saved_spend: float
|
||||
) -> tuple[float | None, float | None]:
|
||||
if turns > 0 and estimated_turns == 0:
|
||||
return None, None
|
||||
return saved_spend, actual_spend + saved_spend
|
||||
|
||||
|
||||
def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
|
||||
return_misses: Final = row.return_turns - row.return_hits
|
||||
baseline_spend: Final = row.spend + row.saved_spend
|
||||
saved_spend, baseline_spend = _savings_cohort(
|
||||
row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend
|
||||
)
|
||||
sessions: Final = row.sessions
|
||||
return AutoRouterBenchmarkTotals(
|
||||
sessions=sessions,
|
||||
|
|
@ -593,11 +606,15 @@ def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
|
|||
avg_session_seconds=row.session_seconds / sessions if sessions else 0.0,
|
||||
avg_tokens_per_session=row.total_tokens / sessions if sessions else 0.0,
|
||||
spend=row.spend,
|
||||
saved_spend=row.saved_spend,
|
||||
savings_estimated_turns=row.savings_estimated_turns,
|
||||
savings_estimated_actual_spend=row.savings_estimated_actual_spend,
|
||||
saved_spend=saved_spend,
|
||||
classifier_cost=row.classifier_cost if row.classifier_cost_recorded_turns == row.turns else None,
|
||||
baseline_spend=baseline_spend,
|
||||
saved_pct=_pct(row.saved_spend, baseline_spend),
|
||||
saved_per_session=row.saved_spend / sessions if sessions else 0.0,
|
||||
saved_pct=_pct(saved_spend, baseline_spend) if saved_spend is not None and baseline_spend is not None else None,
|
||||
saved_per_session=(row.savings_estimated_saved_spend / sessions if sessions else 0.0)
|
||||
if row.savings_estimated_turns == row.turns
|
||||
else None,
|
||||
cache=AutoRouterCacheStats(
|
||||
coverage_pct=_pct(row.covered_turns, row.turns),
|
||||
hit_rate_pct=_pct(row.cache_hits, row.covered_turns),
|
||||
|
|
@ -627,6 +644,8 @@ def _benchmark_group(row: _SessionAggRow) -> AutoRouterBenchmarkGroup:
|
|||
avg_tokens_per_session=totals.avg_tokens_per_session,
|
||||
spend=totals.spend,
|
||||
saved_spend=totals.saved_spend,
|
||||
savings_estimated_turns=totals.savings_estimated_turns,
|
||||
savings_estimated_actual_spend=totals.savings_estimated_actual_spend,
|
||||
classifier_cost=totals.classifier_cost,
|
||||
baseline_spend=totals.baseline_spend,
|
||||
saved_pct=totals.saved_pct,
|
||||
|
|
@ -658,6 +677,9 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
|
|||
total_tokens=sum(row.total_tokens for row in rows),
|
||||
spend=sum(row.spend for row in rows),
|
||||
saved_spend=sum(row.saved_spend for row in rows),
|
||||
savings_estimated_turns=sum(row.savings_estimated_turns for row in rows),
|
||||
savings_estimated_actual_spend=sum(row.savings_estimated_actual_spend for row in rows),
|
||||
savings_estimated_saved_spend=sum(row.savings_estimated_saved_spend for row in rows),
|
||||
classifier_cost=sum(row.classifier_cost for row in rows),
|
||||
classifier_cost_recorded_turns=sum(row.classifier_cost_recorded_turns for row in rows),
|
||||
session_seconds=sum(row.session_seconds for row in rows),
|
||||
|
|
@ -807,6 +829,9 @@ async def get_auto_router_session(
|
|||
raise HTTPException(
|
||||
status_code=404, detail=f"No auto-routed turns recorded for session {session_id!r} under this key"
|
||||
)
|
||||
saved_spend, baseline_spend = _savings_cohort(
|
||||
row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend
|
||||
)
|
||||
return AutoRouterSessionResponse(
|
||||
session_id=session_id,
|
||||
router_name=row.router_name,
|
||||
|
|
@ -814,10 +839,13 @@ async def get_auto_router_session(
|
|||
turns=row.turns,
|
||||
last_model=row.last_model,
|
||||
spend=row.spend,
|
||||
saved_spend=row.saved_spend,
|
||||
baseline_spend=row.spend + row.saved_spend,
|
||||
savings_estimated_turns=row.savings_estimated_turns,
|
||||
savings_estimated_actual_spend=row.savings_estimated_actual_spend,
|
||||
saved_spend=saved_spend,
|
||||
baseline_spend=baseline_spend if row.savings_estimated_turns == row.turns else None,
|
||||
savings_estimated_baseline_spend=baseline_spend,
|
||||
baseline_model=row.baseline_model,
|
||||
baseline_models=row.baseline_models,
|
||||
baseline_models=row.savings_estimated_baseline_models,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ if TYPE_CHECKING:
|
|||
FILE_LIST_CONTINUATION_CHUNK_SIZE: Final = 500
|
||||
|
||||
BATCH_CREATE_HIDDEN_PARAM: Final = "batch_create"
|
||||
LITELLM_EXECUTED_BATCH_ID_PREFIX: Final = "litellm_batch_"
|
||||
|
||||
|
||||
def validate_file_list_limit(limit: int | None) -> None:
|
||||
|
|
@ -179,6 +180,11 @@ def get_batch_id_from_unified_batch_id(file_id: str) -> str:
|
|||
return re.split(r"[;,]", batch_id, maxsplit=1)[0]
|
||||
|
||||
|
||||
def is_litellm_executed_batch(decoded_unified_batch_id: str) -> bool:
|
||||
_, marker, batch_id = decoded_unified_batch_id.partition("llm_batch_id:")
|
||||
return bool(marker) and batch_id.startswith(LITELLM_EXECUTED_BATCH_ID_PREFIX)
|
||||
|
||||
|
||||
def encode_file_id_with_model(file_id: str, model: str, id_type: Literal["file", "batch"] = "file") -> str:
|
||||
"""
|
||||
Encode a file/batch ID with model routing information.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import asyncio
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, BinaryIO, Final, TypedDict, cast, get_args
|
||||
|
||||
import httpx
|
||||
|
|
@ -32,10 +32,15 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
is_managed_cloud_storage_uri,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
|
||||
from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME
|
||||
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
|
||||
from litellm.llms.base_llm.managed_resources.isolation import build_list_page
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.batches_endpoints.litellm_executed_batches import (
|
||||
litellm_executed_provider_of,
|
||||
resolve_litellm_executed_provider,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
_read_request_body,
|
||||
|
|
@ -67,6 +72,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
_is_base64_encoded_unified_file_id,
|
||||
add_internal_model_credentials,
|
||||
apply_team_provider_credentials,
|
||||
authorize_model_for_key,
|
||||
encode_file_id_with_model,
|
||||
extract_file_creation_params,
|
||||
get_authorized_credentials_for_model,
|
||||
|
|
@ -86,7 +92,7 @@ from litellm.proxy.openai_files_endpoints.general_upload_validation import (
|
|||
coerce_optional_str_list_setting,
|
||||
raise_upload_validation_failure,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging, is_known_model
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging, is_known_model
|
||||
from litellm.repositories.table_repositories import ManagedFileRepository
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -99,6 +105,64 @@ from litellm.types.llms.openai import (
|
|||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
||||
def _names_a_litellm_executed_provider(llm_router: Router, candidate: str, team_id: str | None) -> bool:
|
||||
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=candidate, team_id=team_id)
|
||||
return credentials is not None and litellm_executed_provider_of(credentials) is not None
|
||||
|
||||
|
||||
async def _litellm_executed_batch_input_model(
|
||||
llm_router: Router | None,
|
||||
purpose: OpenAIFilesPurpose,
|
||||
model: str | None,
|
||||
target_model_names_list: Sequence[str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
explicit_storage: str | None,
|
||||
) -> str | None:
|
||||
if llm_router is None:
|
||||
return None
|
||||
candidates: Final = (model,) if model is not None else tuple(target_model_names_list)
|
||||
team_id: Final = user_api_key_dict.team_id
|
||||
await asyncio.gather(
|
||||
*(
|
||||
authorize_model_for_key(model_id=candidate, llm_router=llm_router, user_api_key_dict=user_api_key_dict)
|
||||
for candidate in candidates
|
||||
if _names_a_litellm_executed_provider(llm_router, candidate, team_id)
|
||||
)
|
||||
)
|
||||
if explicit_storage is not None:
|
||||
return None
|
||||
providers: Final = await asyncio.gather(
|
||||
*(resolve_litellm_executed_provider(llm_router, candidate, team_id) for candidate in candidates)
|
||||
)
|
||||
executed: Final = tuple(
|
||||
candidate for candidate, provider in zip(candidates, providers, strict=True) if provider is not None
|
||||
)
|
||||
if not executed:
|
||||
return None
|
||||
if purpose != "batch":
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"The server behind {', '.join(executed)} has no Files API, so LiteLLM keeps only batch input "
|
||||
f"files for it and runs the batch itself: upload with purpose=batch; got purpose={purpose}"
|
||||
),
|
||||
type="invalid_request_error",
|
||||
param="purpose",
|
||||
code=400,
|
||||
)
|
||||
if len(candidates) == 1:
|
||||
return executed[0]
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"LiteLLM runs batches for {', '.join(executed)} itself and keeps their input files, so a batch "
|
||||
f"input file can target only that one model; got target_model_names={', '.join(candidates)}"
|
||||
),
|
||||
type="invalid_request_error",
|
||||
param="target_model_names",
|
||||
code=400,
|
||||
)
|
||||
|
||||
|
||||
_MAX_BATCH_FILE_SIZE_MB_ADAPTER: Final = TypeAdapter(int | None)
|
||||
_LISTED_FILES_ADAPTER: Final = TypeAdapter(list[OpenAIFileObject])
|
||||
|
||||
|
|
@ -244,30 +308,41 @@ async def route_create_file(
|
|||
5. Else -> use custom_llm_provider with files_settings
|
||||
"""
|
||||
|
||||
# Handle custom storage backend
|
||||
if target_storage and target_storage != "default":
|
||||
explicit_storage: Final = target_storage if target_storage and target_storage != "default" else None
|
||||
if explicit_storage == LITELLM_DB_STORAGE_BACKEND_NAME:
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"target_storage={LITELLM_DB_STORAGE_BACKEND_NAME} is not a storage a caller can pick: LiteLLM "
|
||||
"chooses it on its own for the batch input files of a model whose batches it runs itself, so "
|
||||
"upload with purpose=batch and name that model instead of target_storage"
|
||||
),
|
||||
type="invalid_request_error",
|
||||
param="target_storage",
|
||||
code=400,
|
||||
)
|
||||
executed_model: Final = await _litellm_executed_batch_input_model(
|
||||
llm_router, purpose, model, target_model_names_list, user_api_key_dict, explicit_storage
|
||||
)
|
||||
storage: Final = explicit_storage or (LITELLM_DB_STORAGE_BACKEND_NAME if executed_model is not None else None)
|
||||
if storage is not None:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.proxy.openai_files_endpoints.storage_backend_service import (
|
||||
StorageBackendFileService,
|
||||
)
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
# Extract file data
|
||||
file_data: Final = extract_file_data(cast(Any, _create_file_request.get("file")))
|
||||
|
||||
# Use storage backend service to handle upload
|
||||
file_object: Final = await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
file_data=file_data,
|
||||
target_storage=target_storage,
|
||||
target_model_names=target_model_names_list,
|
||||
return await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
file_data=extract_file_data(cast(Any, _create_file_request.get("file"))),
|
||||
target_storage=storage,
|
||||
target_model_names=(executed_model,) if executed_model is not None else target_model_names_list,
|
||||
purpose=purpose,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
|
||||
return file_object
|
||||
|
||||
# NEW: Handle model-based routing (no DB required)
|
||||
if model is not None:
|
||||
# Get credentials from model_list via router
|
||||
|
|
@ -848,7 +923,7 @@ async def get_file_content(
|
|||
|
||||
# Check if file is stored in a storage backend (check DB)
|
||||
if hasattr(managed_files_obj, "prisma_client") and getattr(managed_files_obj, "prisma_client", None):
|
||||
prisma_client: Final = getattr(managed_files_obj, "prisma_client")
|
||||
prisma_client: Final[PrismaClient] = getattr(managed_files_obj, "prisma_client")
|
||||
db_file: Final = await ManagedFileRepository(prisma_client).table.find_first(
|
||||
where={"unified_file_id": file_id}
|
||||
)
|
||||
|
|
@ -863,7 +938,7 @@ async def get_file_content(
|
|||
|
||||
try:
|
||||
# Get storage backend (uses same env vars as callback)
|
||||
storage_backend: Final = get_storage_backend(storage_backend_name)
|
||||
storage_backend: Final = get_storage_backend(storage_backend_name, prisma_client=prisma_client)
|
||||
file_content: Final = await storage_backend.download_file(storage_url)
|
||||
|
||||
# Return file content
|
||||
|
|
|
|||
|
|
@ -7,15 +7,16 @@ storage backends (e.g., Azure Blob Storage) and managing associated metadata.
|
|||
|
||||
import base64
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid as uuid_module
|
||||
from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend
|
||||
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
|
||||
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose
|
||||
from litellm.types.utils import SpecialEnums
|
||||
|
||||
|
|
@ -35,21 +36,23 @@ class StorageBackendFileService:
|
|||
async def upload_file_to_storage_backend(
|
||||
file_data: Mapping[str, Any],
|
||||
target_storage: str,
|
||||
target_model_names: list[str],
|
||||
target_model_names: Sequence[str],
|
||||
purpose: OpenAIFilesPurpose,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
prisma_client: PrismaClient | None = None,
|
||||
) -> OpenAIFileObject:
|
||||
"""
|
||||
Upload a file to a storage backend and create a file object.
|
||||
|
||||
Args:
|
||||
file_data: File data dictionary from extract_file_data()
|
||||
target_storage: Storage backend name (e.g., "azure_storage")
|
||||
target_storage: Storage backend name (e.g., "azure_storage", "litellm_db")
|
||||
target_model_names: List of model names for managed files
|
||||
purpose: File purpose (e.g., "user_data", "batch")
|
||||
proxy_logging_obj: Proxy logging object for accessing hooks
|
||||
user_api_key_dict: User API key authentication data
|
||||
prisma_client: The proxy's database client, required by the "litellm_db" backend
|
||||
|
||||
Returns:
|
||||
OpenAIFileObject: Created file object with storage metadata
|
||||
|
|
@ -59,7 +62,7 @@ class StorageBackendFileService:
|
|||
"""
|
||||
# Get storage backend instance
|
||||
try:
|
||||
storage_backend: Final = get_storage_backend(target_storage)
|
||||
storage_backend: Final = get_storage_backend(target_storage, prisma_client=prisma_client)
|
||||
except ValueError as e:
|
||||
raise ProxyException(
|
||||
message=str(e),
|
||||
|
|
@ -103,8 +106,9 @@ class StorageBackendFileService:
|
|||
storage_url=storage_url,
|
||||
)
|
||||
|
||||
# Store in managed files if target_model_names provided
|
||||
if target_model_names:
|
||||
if not target_model_names:
|
||||
return file_object
|
||||
try:
|
||||
await StorageBackendFileService._store_in_managed_files(
|
||||
file_object=file_object,
|
||||
file_data=file_data,
|
||||
|
|
@ -114,9 +118,25 @@ class StorageBackendFileService:
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
await StorageBackendFileService._discard_orphaned_content(storage_backend, storage_url, target_storage)
|
||||
raise
|
||||
return file_object
|
||||
|
||||
@staticmethod
|
||||
async def _discard_orphaned_content(
|
||||
storage_backend: BaseFileStorageBackend, storage_url: str, target_storage: str
|
||||
) -> None:
|
||||
try:
|
||||
await storage_backend.delete_file(storage_url)
|
||||
except Exception as e: # noqa: BLE001 # the metadata failure is what surfaces; a failed cleanup is only logged
|
||||
verbose_proxy_logger.warning(
|
||||
"Could not delete orphaned content at %s on %s after its metadata write failed: %s",
|
||||
storage_url,
|
||||
target_storage,
|
||||
e,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _create_file_object_with_storage_metadata(
|
||||
file_content: bytes,
|
||||
|
|
@ -164,7 +184,7 @@ class StorageBackendFileService:
|
|||
@staticmethod
|
||||
def _create_unified_file_id(
|
||||
file_type: str,
|
||||
target_model_names: list[str],
|
||||
target_model_names: Sequence[str],
|
||||
file_id: str,
|
||||
) -> str:
|
||||
"""
|
||||
|
|
@ -194,7 +214,7 @@ class StorageBackendFileService:
|
|||
async def _store_in_managed_files(
|
||||
file_object: OpenAIFileObject,
|
||||
file_data: Mapping[str, Any],
|
||||
target_model_names: list[str],
|
||||
target_model_names: Sequence[str],
|
||||
target_storage: str,
|
||||
storage_url: str,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
|
|
|
|||
|
|
@ -4787,6 +4787,22 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int:
|
|||
return adopt_model_cost_map(new_model_cost_map)
|
||||
|
||||
|
||||
def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]:
|
||||
"""
|
||||
Translate stored web search interception settings into handler kwargs.
|
||||
|
||||
Drops ``enabled``, which gates the callback rather than configuring it, and
|
||||
drops an ``enabled_providers`` that is not a non-empty list so the handler
|
||||
applies its own default. An empty list otherwise matches no provider at all,
|
||||
and a bare string is iterated one character at a time.
|
||||
"""
|
||||
params: Final = {key: value for key, value in stored.items() if key != "enabled"}
|
||||
providers: Final = params.get("enabled_providers")
|
||||
if not isinstance(providers, list) or not providers:
|
||||
params.pop("enabled_providers", None)
|
||||
return params
|
||||
|
||||
|
||||
def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool:
|
||||
"""
|
||||
Check if an object type should be loaded from the database based on general_settings.supported_db_objects.
|
||||
|
|
@ -4888,6 +4904,7 @@ class ProxyConfig:
|
|||
def __init__(self) -> None:
|
||||
self.config: Mapping[str, object] = MappingProxyType({})
|
||||
self._last_semantic_filter_config: dict[str, object] | None = None
|
||||
self._last_websearch_interception_config: dict[str, object] | None = None
|
||||
self._last_hashicorp_vault_config: dict[str, object] | None = None
|
||||
self._last_cyberark_config: dict[str, object] | None = None # mutable-ok: change-detection cache
|
||||
self._cyberark_boot_env: dict[str, str | None] | None = None # mutable-ok: deployment env snapshot, set once
|
||||
|
|
@ -7709,6 +7726,9 @@ class ProxyConfig:
|
|||
if self._should_load_db_object(object_type="semantic_filter_settings"):
|
||||
await self._init_semantic_filter_settings_in_db(prisma_client=prisma_client)
|
||||
|
||||
if self._should_load_db_object(object_type=SupportedDBObjectType.WEBSEARCH_INTERCEPTION_SETTINGS):
|
||||
await self.init_websearch_interception_settings_in_db(prisma_client=prisma_client)
|
||||
|
||||
if self._should_load_db_object(object_type="config_overrides"):
|
||||
await self._init_hashicorp_vault_config_override(prisma_client=prisma_client)
|
||||
await self._init_cyberark_config_override(prisma_client=prisma_client)
|
||||
|
|
@ -7787,6 +7807,66 @@ class ProxyConfig:
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error initializing semantic filter settings from DB: %s", e)
|
||||
|
||||
async def init_websearch_interception_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
Initialize web search interception settings from database.
|
||||
Called periodically (approximately every 10 seconds) by background task to hot-reload settings across all pods.
|
||||
"""
|
||||
import json
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
try:
|
||||
config_record: Final = await get_config_param(prisma_client, "litellm_settings")
|
||||
|
||||
if config_record is None or config_record.param_value is None:
|
||||
return
|
||||
|
||||
litellm_settings = config_record.param_value
|
||||
if isinstance(litellm_settings, str):
|
||||
litellm_settings = json.loads(litellm_settings)
|
||||
|
||||
websearch_config: Final = litellm_settings.get("websearch_interception_params", None)
|
||||
|
||||
if not isinstance(websearch_config, Mapping):
|
||||
return
|
||||
|
||||
if "enabled" not in websearch_config and self._last_websearch_interception_config is None:
|
||||
verbose_proxy_logger.debug(
|
||||
"Web search interception: stored settings carry no 'enabled' flag and none were applied "
|
||||
"before, so litellm_settings.callbacks keeps ownership of the callback."
|
||||
)
|
||||
return
|
||||
|
||||
enabled: Final = bool(coerce_bool(websearch_config.get("enabled", True)))
|
||||
registered: Final = bool(
|
||||
litellm.logging_callback_manager.get_custom_loggers_for_type(WebSearchInterceptionLogger)
|
||||
)
|
||||
if self._last_websearch_interception_config == websearch_config and registered == enabled:
|
||||
return
|
||||
|
||||
replacement: Final = (
|
||||
WebSearchInterceptionLogger.from_config_yaml(_websearch_handler_params(websearch_config))
|
||||
if enabled
|
||||
else None
|
||||
)
|
||||
|
||||
litellm.logging_callback_manager.remove_callbacks_by_type(litellm.callbacks, WebSearchInterceptionLogger)
|
||||
|
||||
if replacement is not None:
|
||||
litellm.logging_callback_manager.add_litellm_callback(replacement)
|
||||
verbose_proxy_logger.info("Web search interception reinitialized from DB")
|
||||
else:
|
||||
verbose_proxy_logger.info("Web search interception disabled")
|
||||
|
||||
self._last_websearch_interception_config = dict(websearch_config)
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error initializing web search interception settings from DB: %s", e)
|
||||
|
||||
async def _init_sso_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
Initialize SSO settings from database into the router on startup.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.utils import get_custom_url
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets
|
||||
from litellm.types.agents import AgentCard
|
||||
from litellm.types.mcp import MCPPublicServer
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
@ -424,6 +425,14 @@ async def get_complexity_scorer_defaults() -> ComplexityScorerDefaults:
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/public/complexity_router/fuse_presets",
|
||||
response_model=FusePresetCatalog,
|
||||
)
|
||||
async def get_public_fuse_presets() -> FusePresetCatalog:
|
||||
return get_fuse_presets()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/public/litellm_model_cost_map",
|
||||
tags=["public", "model management"],
|
||||
|
|
|
|||
|
|
@ -1107,6 +1107,12 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
|
|||
@@index([team_id, created_at(sort: Desc)])
|
||||
}
|
||||
|
||||
model LiteLLM_ManagedFileContentTable {
|
||||
id String @id @default(uuid())
|
||||
content Bytes
|
||||
created_at DateTime @default(now())
|
||||
}
|
||||
|
||||
model LiteLLM_ManagedVectorStoreTable {
|
||||
id String @id @default(uuid())
|
||||
unified_resource_id String @unique // The base64 encoded unified vector store ID
|
||||
|
|
@ -1545,6 +1551,36 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterBaselineComparison {
|
||||
scope String @id
|
||||
api_key String
|
||||
session_id String
|
||||
router_name String
|
||||
initial_equivalent Boolean
|
||||
revision BigInt @default(0)
|
||||
published_revision BigInt @default(0)
|
||||
history String?
|
||||
attempted_at DateTime?
|
||||
retired Boolean @default(false)
|
||||
updated_at DateTime @default(now())
|
||||
|
||||
@@index([api_key, session_id, router_name], map: "idx_autorouter_baseline_scope")
|
||||
@@index([updated_at], map: "idx_autorouter_baseline_updated")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterBaselineObservation {
|
||||
request_id String @id
|
||||
scope String
|
||||
started_at Float
|
||||
revision BigInt
|
||||
data String
|
||||
publication String?
|
||||
conflicted Boolean @default(false)
|
||||
|
||||
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")
|
||||
@@index([scope, revision, started_at], map: "idx_autorouter_baseline_event_revision")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterSession {
|
||||
api_key String
|
||||
session_id String
|
||||
|
|
@ -1571,6 +1607,10 @@ model LiteLLM_AutoRouterSession {
|
|||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
savings_estimated_baseline_models Json @default("{}")
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
|
|
|
|||
348
litellm/proxy/spend_tracking/baseline_accounting.py
Normal file
348
litellm/proxy/spend_tracking/baseline_accounting.py
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
"""Pure, chronological cache accounting for the recorded baseline comparison.
|
||||
|
||||
Observation collection, pricing and durable publication belong to their existing
|
||||
owners. Replaying these values in event order is independent of callback order.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import groupby
|
||||
from math import isfinite
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from litellm.llms.anthropic.prompt_cache_prediction import CountedBreakpoint, CountedPromptCachePlan
|
||||
from litellm.types.utils import CacheCreationTokenDetails, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
MAX_CACHE_TTL: Final = 3600
|
||||
MAX_CACHE_ENTRIES: Final = 1024
|
||||
|
||||
|
||||
class BaselineObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
version: Literal[3] = 3
|
||||
request_id: str = Field(min_length=1)
|
||||
started_at: float = Field(allow_inf_nan=False, ge=0)
|
||||
available_at: float = Field(allow_inf_nan=False, ge=0)
|
||||
outcome: Literal["complete", "uncertain", "response_cache"]
|
||||
baseline_equivalent: bool
|
||||
usage: Usage | None = None
|
||||
plan: CountedPromptCachePlan | None = None
|
||||
minimum_cache_tokens: int = Field(default=0, ge=0)
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BaselineEstimate:
|
||||
request_id: str
|
||||
reason: str
|
||||
provenance: Literal["observed_identical", "modeled"] | None = None
|
||||
usage: Usage | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CacheEntry:
|
||||
fingerprint: str
|
||||
content_fingerprint: str
|
||||
tokens: int
|
||||
ttl_seconds: int
|
||||
available_at: float
|
||||
expires_at: float
|
||||
uncertain: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BaselineHistory:
|
||||
first_at: float | None = None
|
||||
last_at: float | None = None
|
||||
equivalent: bool = True
|
||||
uncertain_before: float = 0.0
|
||||
entries: tuple[CacheEntry, ...] = ()
|
||||
blocked_until: float = 0.0
|
||||
|
||||
|
||||
def _complete_usage(usage: Usage | None) -> bool:
|
||||
if usage is None or usage.prompt_tokens < 0 or usage.completion_tokens < 0:
|
||||
return False
|
||||
details: Final = usage.prompt_tokens_details
|
||||
if details is None:
|
||||
return False
|
||||
values: Final = (details.text_tokens, details.cached_tokens, details.cache_creation_tokens)
|
||||
if any(value is None or value < 0 for value in values):
|
||||
return False
|
||||
split: Final = details.cache_creation_token_details
|
||||
writes: Final = details.cache_creation_tokens or 0
|
||||
return (
|
||||
usage.total_tokens == usage.prompt_tokens + usage.completion_tokens
|
||||
and sum(value or 0 for value in values) == usage.prompt_tokens
|
||||
and (
|
||||
writes == 0
|
||||
or (
|
||||
split is not None
|
||||
and split.ephemeral_5m_input_tokens is not None
|
||||
and split.ephemeral_1h_input_tokens is not None
|
||||
and min(split.ephemeral_5m_input_tokens, split.ephemeral_1h_input_tokens) >= 0
|
||||
and split.ephemeral_5m_input_tokens + split.ephemeral_1h_input_tokens == writes
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _valid_plan(plan: CountedPromptCachePlan | None) -> bool:
|
||||
if plan is None or plan.total_tokens < 0 or len(plan.breakpoints) > 4:
|
||||
return False
|
||||
return all(
|
||||
marker.fingerprint
|
||||
and marker.content_fingerprint
|
||||
and marker.fingerprint in marker.lookback_fingerprints
|
||||
and marker.content_fingerprint in marker.lookback_content_fingerprints
|
||||
and marker.ttl_seconds in (300, 3600)
|
||||
and 0 <= marker.prefix_tokens <= plan.total_tokens
|
||||
for marker in plan.breakpoints
|
||||
) and all(
|
||||
left.prefix_tokens <= right.prefix_tokens and left.ttl_seconds >= right.ttl_seconds
|
||||
for left, right in zip(plan.breakpoints, plan.breakpoints[1:])
|
||||
)
|
||||
|
||||
|
||||
def _markers(observation: BaselineObservation) -> tuple[CountedBreakpoint, ...]:
|
||||
return (
|
||||
tuple(
|
||||
marker
|
||||
for marker in observation.plan.breakpoints
|
||||
if marker.prefix_tokens >= observation.minimum_cache_tokens
|
||||
)
|
||||
if observation.plan is not None
|
||||
else ()
|
||||
)
|
||||
|
||||
|
||||
def _matches(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: float) -> bool:
|
||||
return entry.available_at <= started < entry.expires_at and any(
|
||||
entry.fingerprint in marker.lookback_fingerprints
|
||||
and entry.tokens <= marker.prefix_tokens
|
||||
and entry.ttl_seconds == marker.ttl_seconds
|
||||
for marker in markers
|
||||
)
|
||||
|
||||
|
||||
def _ambiguous(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: float) -> bool:
|
||||
return entry.available_at <= started < entry.expires_at and any(
|
||||
entry.content_fingerprint in marker.lookback_content_fingerprints
|
||||
and (entry.uncertain or entry.ttl_seconds != marker.ttl_seconds)
|
||||
for marker in markers
|
||||
)
|
||||
|
||||
|
||||
def _usage_with_cache(usage: Usage, total: int, read: int, write_5m: int, write_1h: int) -> Usage:
|
||||
writes: Final = write_5m + write_1h
|
||||
original_details: Final = usage.prompt_tokens_details or PromptTokensDetailsWrapper()
|
||||
details: Final = original_details.model_copy(
|
||||
deep=True,
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"text_tokens": total - read - writes,
|
||||
"cached_tokens": read,
|
||||
"cache_creation_tokens": writes,
|
||||
"cache_write_tokens": writes,
|
||||
"cache_creation_token_details": CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=write_5m,
|
||||
ephemeral_1h_input_tokens=write_1h,
|
||||
),
|
||||
}
|
||||
),
|
||||
)
|
||||
return Usage.model_validate(
|
||||
{ # mutable-ok: Usage only runs its normalizing constructor for a plain dictionary
|
||||
**usage.model_dump(),
|
||||
"prompt_tokens": total,
|
||||
"total_tokens": total + usage.completion_tokens,
|
||||
"prompt_tokens_details": details,
|
||||
"cache_read_input_tokens": read,
|
||||
"cache_creation_input_tokens": writes,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _estimate(history: BaselineHistory, observation: BaselineObservation, equivalent: bool) -> BaselineEstimate:
|
||||
if observation.outcome != "complete" or not _complete_usage(observation.usage):
|
||||
return BaselineEstimate(observation.request_id, observation.reason or observation.outcome)
|
||||
usage: Final = observation.usage
|
||||
if usage is None:
|
||||
return BaselineEstimate(observation.request_id, "missing_usage")
|
||||
if equivalent and observation.baseline_equivalent:
|
||||
return BaselineEstimate(
|
||||
observation.request_id, "identical_baseline_path", "observed_identical", usage.model_copy(deep=True)
|
||||
)
|
||||
if observation.started_at < history.blocked_until:
|
||||
return BaselineEstimate(observation.request_id, "concurrent_uncertainty")
|
||||
plan: Final = observation.plan
|
||||
if not _valid_plan(plan) or plan is None:
|
||||
return BaselineEstimate(observation.request_id, observation.reason or "unsupported_cache_plan")
|
||||
markers: Final = _markers(observation)
|
||||
if any(_ambiguous(entry, markers, observation.started_at) for entry in history.entries):
|
||||
return BaselineEstimate(observation.request_id, "cache_ttl_changed")
|
||||
read: Final = max(
|
||||
(
|
||||
entry.tokens
|
||||
for entry in history.entries
|
||||
if not entry.uncertain and _matches(entry, markers, observation.started_at)
|
||||
),
|
||||
default=0,
|
||||
)
|
||||
end: Final = markers[-1].prefix_tokens if markers else 0
|
||||
if read < end and observation.started_at < history.uncertain_before + max(marker.ttl_seconds for marker in markers):
|
||||
return BaselineEstimate(observation.request_id, "history_unavailable")
|
||||
one_hour: Final = max(
|
||||
(marker.prefix_tokens for marker in markers if marker.ttl_seconds == 3600 and marker.prefix_tokens > read),
|
||||
default=read,
|
||||
)
|
||||
expired: Final = any(
|
||||
entry.expires_at <= observation.started_at
|
||||
and any(entry.fingerprint in marker.lookback_fingerprints for marker in markers)
|
||||
for entry in history.entries
|
||||
)
|
||||
reason: Final = (
|
||||
"cache_prefix_available"
|
||||
if read
|
||||
else "cache_prefix_expired"
|
||||
if expired
|
||||
else "cache_prefix_cold"
|
||||
if markers
|
||||
else "below_cache_minimum"
|
||||
if plan.breakpoints
|
||||
else "no_cache_breakpoints"
|
||||
)
|
||||
return BaselineEstimate(
|
||||
observation.request_id,
|
||||
reason,
|
||||
"modeled",
|
||||
_usage_with_cache(usage, plan.total_tokens, read, end - one_hour, one_hour - read),
|
||||
)
|
||||
|
||||
|
||||
def _writes(history: BaselineHistory, observation: BaselineObservation) -> tuple[CacheEntry, ...]:
|
||||
if (
|
||||
observation.outcome != "complete"
|
||||
or observation.started_at < history.blocked_until
|
||||
or not _complete_usage(observation.usage)
|
||||
or not _valid_plan(observation.plan)
|
||||
):
|
||||
return ()
|
||||
markers: Final = _markers(observation)
|
||||
ambiguous: Final = tuple(entry for entry in history.entries if _ambiguous(entry, markers, observation.started_at))
|
||||
hit: Final = (
|
||||
max(
|
||||
(
|
||||
entry
|
||||
for entry in history.entries
|
||||
if not entry.uncertain and _matches(entry, markers, observation.started_at)
|
||||
),
|
||||
key=lambda entry: entry.tokens,
|
||||
default=None,
|
||||
)
|
||||
if not ambiguous
|
||||
else None
|
||||
)
|
||||
refresh: Final = (
|
||||
(
|
||||
CacheEntry(
|
||||
hit.fingerprint,
|
||||
hit.content_fingerprint,
|
||||
hit.tokens,
|
||||
hit.ttl_seconds,
|
||||
observation.available_at,
|
||||
observation.started_at + hit.ttl_seconds,
|
||||
),
|
||||
)
|
||||
if hit is not None and all(marker.fingerprint != hit.fingerprint for marker in markers)
|
||||
else ()
|
||||
)
|
||||
return (
|
||||
*refresh,
|
||||
*(
|
||||
CacheEntry(
|
||||
marker.fingerprint,
|
||||
marker.content_fingerprint,
|
||||
marker.prefix_tokens,
|
||||
marker.ttl_seconds,
|
||||
observation.available_at,
|
||||
observation.started_at + max((marker.ttl_seconds, *(entry.ttl_seconds for entry in ambiguous))),
|
||||
uncertain=bool(ambiguous),
|
||||
)
|
||||
for marker in markers
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _entry_key(entry: CacheEntry) -> tuple[str, str, int, int, bool]:
|
||||
return entry.fingerprint, entry.content_fingerprint, entry.tokens, entry.ttl_seconds, entry.uncertain
|
||||
|
||||
|
||||
def _compact_entries(entries: tuple[CacheEntry, ...], started: float) -> tuple[CacheEntry, ...]:
|
||||
ordered: Final = sorted((entry for entry in entries if entry.expires_at >= started - MAX_CACHE_TTL), key=_entry_key)
|
||||
return tuple(
|
||||
retained
|
||||
for _, values in groupby(ordered, key=_entry_key)
|
||||
for group in (tuple(values),)
|
||||
for retained in (
|
||||
max(
|
||||
(entry for entry in group if entry.available_at <= started),
|
||||
key=lambda entry: entry.expires_at,
|
||||
default=None,
|
||||
),
|
||||
*(entry for entry in group if entry.available_at > started),
|
||||
)
|
||||
if retained is not None
|
||||
)
|
||||
|
||||
|
||||
def advance_baseline_history(
|
||||
history: BaselineHistory,
|
||||
simultaneous: Sequence[BaselineObservation],
|
||||
) -> tuple[BaselineHistory, tuple[BaselineEstimate, ...]]:
|
||||
"""Apply one request-start timestamp; ties cannot manufacture initial equality.
|
||||
|
||||
The storage owner groups and orders observations before calling this function.
|
||||
Equal timestamps are evaluated against the same preceding cache snapshot.
|
||||
"""
|
||||
if not simultaneous:
|
||||
return history, ()
|
||||
started: Final = simultaneous[0].started_at
|
||||
valid_order: Final = (
|
||||
isfinite(started)
|
||||
and all(item.started_at == started and item.available_at >= started for item in simultaneous)
|
||||
and (history.last_at is None or started > history.last_at)
|
||||
)
|
||||
if not valid_order:
|
||||
return history, tuple(BaselineEstimate(item.request_id, "invalid_observation_order") for item in simultaneous)
|
||||
first: Final = started if history.first_at is None else history.first_at
|
||||
uncertain: Final = max(history.uncertain_before, first)
|
||||
relevant: Final = tuple(item for item in simultaneous if item.outcome != "response_cache")
|
||||
equivalent: Final = history.equivalent and all(item.baseline_equivalent for item in relevant)
|
||||
before: Final = BaselineHistory(
|
||||
first, history.last_at, equivalent, uncertain, history.entries, history.blocked_until
|
||||
)
|
||||
estimates: Final = tuple(_estimate(before, item, equivalent) for item in simultaneous)
|
||||
invalidated: Final = any(
|
||||
item.outcome != "complete" or not _complete_usage(item.usage) or not _valid_plan(item.plan) for item in relevant
|
||||
)
|
||||
blocked: Final = max((history.blocked_until, *(item.available_at for item in relevant if invalidated)))
|
||||
entries: Final = _compact_entries(
|
||||
() if invalidated else (*history.entries, *(entry for item in relevant for entry in _writes(before, item))),
|
||||
started,
|
||||
)
|
||||
overflow: Final = len(entries) > MAX_CACHE_ENTRIES
|
||||
return BaselineHistory(
|
||||
first_at=first,
|
||||
last_at=started,
|
||||
equivalent=equivalent,
|
||||
uncertain_before=max(started, blocked) if invalidated or overflow else uncertain,
|
||||
entries=() if overflow else entries,
|
||||
blocked_until=blocked,
|
||||
), estimates
|
||||
|
|
@ -10,7 +10,11 @@ have been aggregated across models.
|
|||
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Final, NamedTuple
|
||||
from math import isclose, isfinite
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, NamedTuple
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -65,7 +69,7 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model
|
|||
return None
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings
|
||||
except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to an unavailable estimate
|
||||
verbose_proxy_logger.debug(
|
||||
"savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e
|
||||
)
|
||||
|
|
@ -118,6 +122,68 @@ class PricingBasis(NamedTuple):
|
|||
_STANDARD_RATES: Final = PricingBasis()
|
||||
|
||||
|
||||
class BaselineCostSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
model: str
|
||||
provider: str
|
||||
prices: ModelInfo | None
|
||||
basis: PricingBasis = _STANDARD_RATES
|
||||
actual_spend: float = Field(allow_inf_nan=False, ge=0)
|
||||
actual_token_cost: float | None = Field(default=None, allow_inf_nan=False, ge=0)
|
||||
classifier_cost: float = Field(default=0.0, allow_inf_nan=False, ge=0)
|
||||
|
||||
|
||||
def baseline_cost_snapshot(
|
||||
model: str,
|
||||
prices: ModelInfo | None,
|
||||
actual_spend: float,
|
||||
cost_breakdown: Mapping[str, object] | None,
|
||||
routing_decision: Mapping[str, object] | None,
|
||||
) -> BaselineCostSnapshot:
|
||||
return BaselineCostSnapshot(
|
||||
model=model,
|
||||
provider="anthropic",
|
||||
prices=prices,
|
||||
actual_spend=actual_spend,
|
||||
basis=_pricing_basis(cost_breakdown),
|
||||
actual_token_cost=_recorded_token_cost(cost_breakdown),
|
||||
classifier_cost=classifier_cost_from_decision(routing_decision) or 0.0,
|
||||
)
|
||||
|
||||
|
||||
class BaselineCosts(NamedTuple):
|
||||
actual: float
|
||||
baseline: float
|
||||
|
||||
@property
|
||||
def savings(self) -> float:
|
||||
return self.baseline - self.actual
|
||||
|
||||
|
||||
def price_baseline_comparison(
|
||||
snapshot: BaselineCostSnapshot,
|
||||
baseline_usage: Usage | None,
|
||||
provenance: Literal["observed_identical", "modeled"] | None,
|
||||
) -> BaselineCosts | None:
|
||||
if baseline_usage is None or provenance is None:
|
||||
return None
|
||||
actual: Final = snapshot.actual_spend + snapshot.classifier_cost
|
||||
if provenance == "observed_identical":
|
||||
return BaselineCosts(actual=actual, baseline=snapshot.actual_spend)
|
||||
if snapshot.prices is None or snapshot.actual_token_cost is None:
|
||||
return None
|
||||
token_cost: Final = _cost_of_usage(
|
||||
_ModelIdentity(snapshot.model, snapshot.provider), baseline_usage, snapshot.prices, snapshot.basis
|
||||
)
|
||||
if token_cost is None or not isfinite(token_cost) or token_cost < 0:
|
||||
return None
|
||||
baseline: Final = snapshot.actual_spend + token_cost - snapshot.actual_token_cost
|
||||
if not isfinite(baseline) or baseline < 0:
|
||||
return None
|
||||
return BaselineCosts(actual=actual, baseline=baseline)
|
||||
|
||||
|
||||
def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis:
|
||||
"""The basis recorded on a request, defaulting to standard rates when absent.
|
||||
|
||||
|
|
@ -225,56 +291,16 @@ def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bo
|
|||
)
|
||||
|
||||
|
||||
def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage:
|
||||
"""The same request as a single-model baseline would have met it.
|
||||
|
||||
The baseline is one model serving every turn, so whether it had this prompt cached
|
||||
is simply whether the conversation was already underway. On a continuing
|
||||
conversation it wrote the prompt on an earlier turn and would only read it now, so
|
||||
the cache tokens move into the read bucket and whatever this request paid to write
|
||||
counts against the saving; that write is what switching models costs.
|
||||
|
||||
On a conversation's first turn nothing was cached anywhere, for any model. The
|
||||
baseline would have written the same prompt, so the cache buckets stay where they are
|
||||
and both arms carry the write at their own rates, unless the baseline has no rate for
|
||||
a bucket, in which case those tokens are its plain input. Charging the write to this case
|
||||
too, which is all a single rollup row can support, understates a first turn to a
|
||||
few percent of its value and can render a profitable route as a loss.
|
||||
|
||||
A continuing turn that mostly read from cache is the third case: the selected model
|
||||
was already warm, so it is the one that has been serving this conversation and the
|
||||
baseline's cache holds exactly what its does. The tokens written are the turn's own
|
||||
growth, new to every model, and the baseline would have paid to write them too.
|
||||
Moving them would forgive the baseline a write it really owes and shrink the
|
||||
reported saving. "Mostly read" rather than "read anything" on purpose: a switch onto
|
||||
a model holding a small prefix of this prompt still writes most of it, and must keep
|
||||
counting that write against the saving.
|
||||
|
||||
Only the cache buckets move. Every other field the request was priced on travels
|
||||
through untouched, audio and image and video counts among them, because the baseline
|
||||
is this same request served by a model that happened to be warm; naming the fields to
|
||||
keep instead would price the baseline on a request that never ran, and would go stale
|
||||
the next time a priced field is added.
|
||||
"""
|
||||
def _baseline_usage(usage: Usage, baseline_info: ModelInfo | None = None) -> Usage:
|
||||
cache_read, cache_creation = _cache_token_split(usage)
|
||||
details: Final = usage.prompt_tokens_details
|
||||
if details is None or (cache_read <= 0 and cache_creation <= 0):
|
||||
return usage
|
||||
|
||||
# The tokens this request paid to write move into the cached count and the creation
|
||||
# charge is dropped: on one model that cache was already warm, so the baseline would
|
||||
# have read them rather than paying to create them. The 5m/1h breakdown goes with
|
||||
# them; left behind it re-charges the write.
|
||||
warm: Final = conversation_continuing and cache_creation > 0 and cache_read <= cache_creation
|
||||
reads = cache_read + cache_creation if warm else cache_read
|
||||
writes = 0 if warm else cache_creation
|
||||
|
||||
prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info)
|
||||
reads = reads if prices_reads else 0
|
||||
writes = writes if prices_writes else 0
|
||||
reads: Final = cache_read if prices_reads else 0
|
||||
writes: Final = cache_creation if prices_writes else 0
|
||||
if (reads, writes) == (cache_read, cache_creation):
|
||||
return usage
|
||||
|
||||
other_modalities: Final = sum(
|
||||
(getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens")
|
||||
)
|
||||
|
|
@ -309,64 +335,47 @@ def compute_autorouter_savings(
|
|||
cost_breakdown: Mapping[str, object] | None = None,
|
||||
baseline_deployment_id: str | None = None,
|
||||
selected_deployment_id: str | None = None,
|
||||
) -> float:
|
||||
"""Net dollars the router saved, or cost, by serving this request on ``selected_model``.
|
||||
|
||||
Signed on purpose. Switching models leaves the new one with a cold cache, so the
|
||||
request pays a cache-creation charge that staying on one model would not have
|
||||
incurred; when that charge outweighs the cheaper rates, routing lost money and the
|
||||
dashboard has to be able to say so. Zero when both sides resolve to the same
|
||||
deployment, or when either cannot be resolved or priced.
|
||||
|
||||
Only one side of this subtraction is a counterfactual. What the request cost on the
|
||||
model that served it is a number the operator was actually billed, and the cost
|
||||
calculator already wrote it down, so ``cost_breakdown`` is read rather than
|
||||
re-derived. Recomputing it means restating every pricing dimension the biller
|
||||
applied, and each one omitted is a silent disagreement with the ``spend`` column
|
||||
beside it; a request billed at a priority tier recomputed at standard rates reads as
|
||||
half its real cost.
|
||||
|
||||
The baseline has no such record, since it never ran, so it is priced through the same
|
||||
cost engine on the basis the biller used for this request. An operator running that
|
||||
one model instead of the router would have sent this request to the same tier and the
|
||||
same region, because both are properties of the request and the deployment's
|
||||
contract, not of which model the router happened to pick.
|
||||
|
||||
``conversation_continuing`` says whether the baseline would already have had this
|
||||
prompt cached. It defaults to True because that is the conservative reading: a
|
||||
request whose shape the router could not determine is charged the write and
|
||||
under-claims rather than inflating a savings figure.
|
||||
"""
|
||||
# No provider argument for the baseline on purpose: it arrives from the routing
|
||||
# metadata as a single self-describing string, already qualified by the auto-router,
|
||||
# so there is no second field that could disagree with it.
|
||||
baseline_usage: Usage | None = None,
|
||||
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
|
||||
) -> float | None:
|
||||
"""Price established baseline usage; conversation shape cannot establish cache warmth."""
|
||||
baseline: Final = _resolve_model(baseline_model, None)
|
||||
selected: Final = _resolve_model(selected_model, selected_provider)
|
||||
if baseline is None or selected is None:
|
||||
return 0.0
|
||||
same_target: Final = (
|
||||
baseline_deployment_id == selected_deployment_id
|
||||
if baseline_deployment_id and selected_deployment_id
|
||||
else baseline == selected
|
||||
)
|
||||
if same_target:
|
||||
return 0.0
|
||||
return None
|
||||
if baseline_usage is None and any(_cache_token_split(usage)):
|
||||
return None
|
||||
basis: Final = _pricing_basis(cost_breakdown)
|
||||
effective_baseline_info: Final = baseline_info if baseline_info is not None else _model_info(baseline)
|
||||
modeled_usage: Final = baseline_usage if baseline_usage is not None else usage
|
||||
baseline_cost: Final = _cost_of_usage(
|
||||
baseline,
|
||||
_baseline_usage(usage, conversation_continuing, effective_baseline_info),
|
||||
effective_baseline_info,
|
||||
basis,
|
||||
baseline, _baseline_usage(modeled_usage, effective_baseline_info), effective_baseline_info, basis
|
||||
)
|
||||
recorded_selected_cost: Final = _recorded_token_cost(cost_breakdown)
|
||||
selected_cost: Final = (
|
||||
recorded_selected_cost
|
||||
if recorded_selected_cost is not None
|
||||
else _cost_of_usage(selected, usage, selected_info, basis)
|
||||
)
|
||||
# Falls back to pricing the request only when the biller recorded nothing, which is
|
||||
# every row written before the breakdown carried its basis.
|
||||
selected_cost = _recorded_token_cost(cost_breakdown)
|
||||
if selected_cost is None:
|
||||
selected_cost = _cost_of_usage(selected, usage, selected_info, basis)
|
||||
if baseline_cost is None or selected_cost is None:
|
||||
return 0.0
|
||||
return baseline_cost - selected_cost
|
||||
return None
|
||||
if baseline_provenance == "observed_initial":
|
||||
same_prices: Final = effective_baseline_info == (
|
||||
selected_info if selected_info is not None else _model_info(selected)
|
||||
)
|
||||
equivalent: Final = (
|
||||
baseline_usage is not None
|
||||
and baseline_usage == usage
|
||||
and baseline == selected
|
||||
and bool(baseline_deployment_id)
|
||||
and baseline_deployment_id == selected_deployment_id
|
||||
and same_prices
|
||||
and recorded_selected_cost is not None
|
||||
and isclose(baseline_cost, recorded_selected_cost, rel_tol=1e-9, abs_tol=1e-12)
|
||||
)
|
||||
return 0.0 if equivalent else None
|
||||
difference: Final = baseline_cost - selected_cost
|
||||
return difference if isfinite(difference) else None
|
||||
|
||||
|
||||
def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None:
|
||||
|
|
@ -463,11 +472,23 @@ def _proxy_llm_router() -> "Router | None":
|
|||
|
||||
def _numeric_savings(value: object) -> float | None:
|
||||
"""``value`` as a recorded savings figure, or ``None`` when it is not one."""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value):
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
|
||||
def recorded_estimated_autorouter_savings(metadata: Mapping[str, object]) -> float | None:
|
||||
estimate: Final = metadata.get("autorouter_savings_estimate")
|
||||
if (
|
||||
not isinstance(estimate, Mapping)
|
||||
or type(estimate.get("version")) is not int
|
||||
or estimate.get("version") not in (1, 2, 3)
|
||||
or estimate.get("status") != "estimated"
|
||||
):
|
||||
return None
|
||||
return _numeric_savings(metadata.get("autorouter_savings"))
|
||||
|
||||
|
||||
def classifier_cost_from_decision(routing_decision: Mapping[str, object] | None) -> float | None:
|
||||
"""The LLM-classifier cost a routing decision recorded, or ``None`` when it holds none.
|
||||
|
||||
|
|
@ -490,22 +511,10 @@ def autorouter_savings_for_request(
|
|||
model_id: str | None = None,
|
||||
llm_router: "Callable[[], Router | None] | None" = None,
|
||||
cost_breakdown: Mapping[str, object] | None = None,
|
||||
baseline_usage: Usage | None = None,
|
||||
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
|
||||
) -> float | None:
|
||||
"""Auto-router savings for one request, net of the classifier call that routed it,
|
||||
or ``None`` when the driver is off.
|
||||
|
||||
``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a
|
||||
figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a
|
||||
real figure for a routed request whose baseline resolved to the served deployment.
|
||||
Never raises: pricing failures inside degrade to zero, and the driver-off cases
|
||||
return ``None``, so this is safe on the logging path where a raise would fail the
|
||||
request's logging.
|
||||
|
||||
The classifier deduction lives here, at the figure's one computation owner, rather
|
||||
than in any reader: the stamped ``autorouter_savings`` is then already net, so the
|
||||
session rollup, the daily tables and every logging consumer agree without each
|
||||
re-deriving the deduction, and the recorded-figure-wins path cannot deduct twice.
|
||||
"""
|
||||
"""Return net savings for established usage, or None when the estimate is unavailable."""
|
||||
usage: Final = _usage_from_spend_log(usage_object)
|
||||
if usage is None or not model:
|
||||
return None
|
||||
|
|
@ -522,15 +531,16 @@ def autorouter_savings_for_request(
|
|||
selected_model=model,
|
||||
selected_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
# Absent means the router never recorded a shape, which is the conservative
|
||||
# reading: charge the cache write rather than claim a first turn's saving.
|
||||
conversation_continuing=decision.get("conversation_continuing") is not False,
|
||||
selected_info=_effective_model_info(router_instance, model_id, model or ""),
|
||||
baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
|
||||
cost_breakdown=cost_breakdown,
|
||||
baseline_deployment_id=baseline_id,
|
||||
selected_deployment_id=model_id,
|
||||
baseline_usage=baseline_usage,
|
||||
baseline_provenance=baseline_provenance,
|
||||
)
|
||||
if gross is None:
|
||||
return None
|
||||
classifier_cost: Final = classifier_cost_from_decision(decision)
|
||||
return gross if classifier_cost is None else gross - classifier_cost
|
||||
|
||||
|
|
@ -542,6 +552,8 @@ def autorouter_savings_for_logging_payload(
|
|||
model_id: str | None,
|
||||
usage_object: Mapping[str, object] | None,
|
||||
cost_breakdown: Mapping[str, object] | None,
|
||||
baseline_usage: Usage | None = None,
|
||||
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
|
||||
) -> float | None:
|
||||
"""The figure the logging payload records for a request, or ``None`` when none should be.
|
||||
|
||||
|
|
@ -561,6 +573,8 @@ def autorouter_savings_for_logging_payload(
|
|||
model_id=model_id,
|
||||
llm_router=_proxy_llm_router,
|
||||
cost_breakdown=cost_breakdown,
|
||||
baseline_usage=baseline_usage,
|
||||
baseline_provenance=baseline_provenance,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -575,6 +589,7 @@ def compute_savings_spend(
|
|||
llm_router: "Callable[[], Router | None] | None" = None,
|
||||
cost_breakdown: Mapping[str, object] | None = None,
|
||||
recorded_autorouter_savings: object = None,
|
||||
recorded_autorouter_savings_estimate: Mapping[str, object] | None = None,
|
||||
billed_at: datetime | str | None = None,
|
||||
) -> SavingsSpend:
|
||||
"""
|
||||
|
|
@ -604,11 +619,9 @@ def compute_savings_spend(
|
|||
figure is normally the smaller of the two, being a subset of the same requests, but
|
||||
not always: a request that only writes cache and never reads it has negative net
|
||||
savings, and dropping such a request from the attributed figure can lift it above
|
||||
the total. Auto-router savings compare the
|
||||
served ``model`` against the counterfactual baseline the router recorded on
|
||||
its ``routing_decision``, and are zero unless the two differ. That record
|
||||
also says whether the conversation was already underway, which is what tells
|
||||
a mid-conversation switch from a first turn.
|
||||
the total. Auto-router savings compare established baseline usage against the
|
||||
recorded selected-model cost. Versioned unknown estimates contribute no dollars
|
||||
to this subtotal and are excluded from the separately reported coverage cohort.
|
||||
|
||||
``llm_router`` is passed as a provider rather than a router because every spend write
|
||||
calls this and only auto-routed ones need one, so looking it up eagerly at the call
|
||||
|
|
@ -653,10 +666,21 @@ def compute_savings_spend(
|
|||
|
||||
# The figure the logging path recorded wins, before the usage gate on purpose: a row
|
||||
# whose usage no longer parses still carries the number computed when it did.
|
||||
recorded_savings: Final = _numeric_savings(recorded_autorouter_savings)
|
||||
recorded_savings: Final = (
|
||||
recorded_estimated_autorouter_savings(
|
||||
MappingProxyType(
|
||||
{
|
||||
"autorouter_savings": recorded_autorouter_savings,
|
||||
"autorouter_savings_estimate": recorded_autorouter_savings_estimate,
|
||||
}
|
||||
)
|
||||
)
|
||||
if recorded_autorouter_savings_estimate is not None
|
||||
else _numeric_savings(recorded_autorouter_savings)
|
||||
)
|
||||
autorouter: Final = (
|
||||
recorded_savings
|
||||
if recorded_savings is not None
|
||||
if recorded_savings is not None or recorded_autorouter_savings_estimate is not None
|
||||
else autorouter_savings_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from functools import reduce
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, JsonValue
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -137,7 +137,15 @@ def _get_router_metadata_for_spend_log(
|
|||
)
|
||||
|
||||
|
||||
_STAMPED_METADATA_KEYS: Final = frozenset(("router_metadata", "azure_spillover"))
|
||||
_STAMPED_METADATA_KEYS: Final = frozenset(
|
||||
(
|
||||
"router_metadata",
|
||||
"azure_spillover",
|
||||
"autorouter_savings",
|
||||
"autorouter_savings_estimate",
|
||||
"autorouter_baseline_observation",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _get_spend_logs_metadata(
|
||||
|
|
@ -156,6 +164,8 @@ def _get_spend_logs_metadata(
|
|||
cost_breakdown: CostBreakdown | None = None,
|
||||
litellm_call_id: str | None = None,
|
||||
autorouter_savings: float | None = None,
|
||||
autorouter_savings_estimate: Mapping[str, JsonValue] | None = None,
|
||||
autorouter_baseline_observation: str | None = None,
|
||||
router_metadata: SpendLogsRouterMetadata | None = None,
|
||||
azure_spillover: AzureSpillover | None = None,
|
||||
) -> SpendLogsMetadata:
|
||||
|
|
@ -196,6 +206,8 @@ def _get_spend_logs_metadata(
|
|||
cost_breakdown=None,
|
||||
compression_savings=None,
|
||||
autorouter_savings=autorouter_savings,
|
||||
autorouter_savings_estimate=autorouter_savings_estimate,
|
||||
autorouter_baseline_observation=autorouter_baseline_observation,
|
||||
litellm_gateway_injected_cache=None,
|
||||
litellm_call_id=litellm_call_id,
|
||||
router_metadata=router_metadata,
|
||||
|
|
@ -207,7 +219,12 @@ def _get_spend_logs_metadata(
|
|||
|
||||
# Filter the metadata dictionary to include only the specified keys
|
||||
clean_metadata: Final = SpendLogsMetadata(
|
||||
**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key not in _STAMPED_METADATA_KEYS},
|
||||
**MappingProxyType(
|
||||
{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key not in _STAMPED_METADATA_KEYS}
|
||||
),
|
||||
autorouter_savings=autorouter_savings,
|
||||
autorouter_savings_estimate=autorouter_savings_estimate,
|
||||
autorouter_baseline_observation=autorouter_baseline_observation,
|
||||
router_metadata=router_metadata,
|
||||
azure_spillover=azure_spillover,
|
||||
)
|
||||
|
|
@ -231,7 +248,6 @@ def _get_spend_logs_metadata(
|
|||
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
|
||||
clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms
|
||||
clean_metadata["cost_breakdown"] = cost_breakdown
|
||||
clean_metadata["autorouter_savings"] = autorouter_savings
|
||||
clean_metadata["litellm_call_id"] = litellm_call_id
|
||||
|
||||
return clean_metadata
|
||||
|
|
@ -660,6 +676,16 @@ def get_logging_payload(
|
|||
autorouter_savings=(
|
||||
standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None
|
||||
),
|
||||
autorouter_savings_estimate=(
|
||||
standard_logging_payload.get("autorouter_savings_estimate")
|
||||
if standard_logging_payload is not None
|
||||
else None
|
||||
),
|
||||
autorouter_baseline_observation=(
|
||||
standard_logging_payload.get("autorouter_baseline_observation")
|
||||
if standard_logging_payload is not None
|
||||
else None
|
||||
),
|
||||
litellm_call_id=litellm_call_id,
|
||||
router_metadata=_get_router_metadata_for_spend_log(
|
||||
metadata=metadata,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from collections import Counter
|
|||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
Annotated,
|
||||
Final,
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
|
|
@ -477,6 +478,72 @@ class MCPToolSearchSettingsResponse(SettingsResponse):
|
|||
"""Response model for native MCP tool search settings"""
|
||||
|
||||
|
||||
class WebSearchInterceptionSettings(BaseModel):
|
||||
"""Configuration for server-side web search interception"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Serve web search tool calls from a configured search tool instead of passing them upstream",
|
||||
)
|
||||
|
||||
enabled_providers: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="LLM providers to intercept for (e.g. 'bedrock', 'vertex_ai'). Empty intercepts Bedrock only.",
|
||||
)
|
||||
|
||||
search_tool_name: str | None = Field(
|
||||
default=None,
|
||||
description="Name of the configured search tool to run searches through. Empty uses the first one available.",
|
||||
)
|
||||
|
||||
max_agentic_loops: int | None = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
description="How many follow-up model calls one intercepted request may chain. Empty applies the default of 3.",
|
||||
)
|
||||
|
||||
|
||||
class WebSearchInterceptionSettingsResponse(SettingsResponse):
|
||||
"""Response model for web search interception settings"""
|
||||
|
||||
|
||||
def _with_websearch_enabled_resolved(config: Mapping[str, object]) -> dict[str, object]:
|
||||
"""
|
||||
Answer with the stored flag when there is one, and only otherwise with what
|
||||
this process is running.
|
||||
|
||||
A stored flag is the cluster's own answer, so it is the same on every pod and
|
||||
is safe for the page to send back on save. Deriving the answer from this
|
||||
process instead would report off on a pod that has not polled yet, and the
|
||||
next save would persist that as a cluster-wide off. Without a stored flag the
|
||||
only available answer is local: litellm_settings.callbacks activates
|
||||
interception without storing one, and a write through the generic config
|
||||
endpoint can drop the flag from a block that is still live. Reporting the
|
||||
field default there would claim the feature is off while it serves.
|
||||
"""
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
litellm_settings: Final[Mapping[str, object]] = _as_settings_section(config.get("litellm_settings"))
|
||||
stored: Final[Mapping[str, object]] = _as_settings_section(litellm_settings.get("websearch_interception_params"))
|
||||
if "enabled" in stored:
|
||||
return dict(config)
|
||||
|
||||
resolved: Final = {
|
||||
**stored,
|
||||
"enabled": bool(litellm.logging_callback_manager.get_custom_loggers_for_type(WebSearchInterceptionLogger)),
|
||||
}
|
||||
return {
|
||||
**config,
|
||||
"litellm_settings": {**litellm_settings, "websearch_interception_params": resolved},
|
||||
}
|
||||
|
||||
|
||||
def _as_settings_section(value: object) -> Mapping[str, object]:
|
||||
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({})
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/allowed_ips",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
|
|
@ -875,7 +942,13 @@ async def update_default_team_member_budget(teams: list[NewUserRequestTeam], use
|
|||
|
||||
|
||||
async def _update_litellm_setting(
|
||||
settings: DefaultInternalUserParams | DefaultTeamSSOParams | MCPSemanticFilterSettings | MCPToolSearchSettings,
|
||||
settings: (
|
||||
DefaultInternalUserParams
|
||||
| DefaultTeamSSOParams
|
||||
| MCPSemanticFilterSettings
|
||||
| MCPToolSearchSettings
|
||||
| WebSearchInterceptionSettings
|
||||
),
|
||||
settings_key: str,
|
||||
success_message: str,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
@ -1399,6 +1472,77 @@ async def update_mcp_semantic_filter_settings(
|
|||
return result
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/websearch_interception_settings",
|
||||
tags=["Settings"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=WebSearchInterceptionSettingsResponse,
|
||||
)
|
||||
async def get_websearch_interception_settings(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
"""
|
||||
Get web search interception configuration.
|
||||
|
||||
Returns the current settings plus their schema, for the Admin UI to render.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_config
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "Database not connected. Please connect a database."},
|
||||
)
|
||||
|
||||
config: Final = await proxy_config.get_config()
|
||||
|
||||
return await _get_settings_with_schema(
|
||||
settings_key="websearch_interception_params",
|
||||
settings_class=WebSearchInterceptionSettings,
|
||||
config=_with_websearch_enabled_resolved(config),
|
||||
)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/update/websearch_interception_settings",
|
||||
tags=["Settings"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def update_websearch_interception_settings(
|
||||
settings: WebSearchInterceptionSettings,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
"""
|
||||
Update web search interception settings in database.
|
||||
|
||||
Settings will be picked up by all pods within approximately 10 seconds via background polling.
|
||||
"""
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Only proxy admins can update web search interception settings.",
|
||||
)
|
||||
|
||||
result: Final = await _update_litellm_setting(
|
||||
settings=settings,
|
||||
settings_key="websearch_interception_params",
|
||||
success_message=(
|
||||
"Web search interception settings updated successfully. "
|
||||
"Changes will be applied across all pods within 10 seconds."
|
||||
),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_config
|
||||
|
||||
if prisma_client is not None:
|
||||
await proxy_config.init_websearch_interception_settings_in_db(prisma_client=prisma_client)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning("Failed to reinitialize web search interception settings immediately: %s", e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/mcp_tool_search_settings",
|
||||
tags=["Settings"], # mutable-ok: FastAPI's route decorator only accepts a list
|
||||
|
|
|
|||
|
|
@ -248,6 +248,7 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
|
||||
from litellm.models.team import LiteLLM_TeamTableCachedObj
|
||||
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
|
||||
from litellm.proxy.db.baseline_accounting import BaselineAccountingRecord
|
||||
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
|
||||
|
|
@ -470,12 +471,16 @@ def _record_raising_guardrail(request_data: Mapping[str, object], callback: obje
|
|||
|
||||
|
||||
class _UpstreamStreamBoundary(Generic[_T]):
|
||||
__slots__ = ("_upstream", "failure")
|
||||
__slots__ = ("_source", "_upstream", "failure")
|
||||
|
||||
def __init__(self, upstream: AsyncIterable[_T]) -> None:
|
||||
self._source: Final = upstream
|
||||
self._upstream: Final = upstream.__aiter__()
|
||||
self.failure: BaseException | None = None
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
return getattr(self._source, name)
|
||||
|
||||
def __aiter__(self) -> "_UpstreamStreamBoundary[_T]":
|
||||
return self
|
||||
|
||||
|
|
@ -3036,6 +3041,10 @@ class ProxyLogging:
|
|||
Otherwise, returns None and the original exception is used.
|
||||
"""
|
||||
|
||||
logging_obj: Final[object] = request_data.get("litellm_logging_obj") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # legacy request data is narrowed to Logging below
|
||||
if isinstance(logging_obj, Logging) and logging_obj.baseline_cache_context is not None:
|
||||
await logging_obj.invalidate_baseline_cache_estimate("failed_request", completed=True)
|
||||
|
||||
### ALERTING ###
|
||||
await self.update_request_status(litellm_call_id=request_data.get("litellm_call_id", ""), status="fail")
|
||||
if AlertType.llm_exceptions in self.alert_types and not _is_client_error_exception(original_exception):
|
||||
|
|
@ -4188,6 +4197,10 @@ class PrismaClient:
|
|||
http_client: "HttpConfig | None" = None,
|
||||
):
|
||||
## init logging object
|
||||
self.baseline_accounting_transactions: list[
|
||||
BaselineAccountingRecord
|
||||
] = [] # mutable-ok: locked background queue
|
||||
self.baseline_accounting_lock: Final = asyncio.Lock()
|
||||
self.proxy_logging_obj = proxy_logging_obj
|
||||
self.token_auth: DatabaseTokenAuth | None = resolve_database_token_auth()
|
||||
verbose_proxy_logger.debug("Creating Prisma Client..")
|
||||
|
|
@ -7156,7 +7169,15 @@ async def _total_queued_spend_transactions(prisma_client: PrismaClient) -> int:
|
|||
autorouter_queue_size: Final = len(prisma_client.autorouter_turn_transactions)
|
||||
from litellm.proxy.db.shadow_eval_funnel import pending_shadow_eval_funnel_events
|
||||
|
||||
return spend_queue_size + tool_queue_size + autorouter_queue_size + pending_shadow_eval_funnel_events()
|
||||
async with prisma_client.baseline_accounting_lock:
|
||||
baseline_queue_size: Final = len(prisma_client.baseline_accounting_transactions)
|
||||
return (
|
||||
spend_queue_size
|
||||
+ tool_queue_size
|
||||
+ autorouter_queue_size
|
||||
+ baseline_queue_size
|
||||
+ pending_shadow_eval_funnel_events()
|
||||
)
|
||||
|
||||
|
||||
async def update_daily_tag_spend(
|
||||
|
|
@ -7221,7 +7242,10 @@ async def update_spend_logs_job(
|
|||
# Atomically pop batch from queue. The tool usage queue counts toward the
|
||||
# emptiness check: a spend-log write failure aborts a run before the tool
|
||||
# drain below, and those entries must not strand once the spend queue drains.
|
||||
from litellm.proxy.db.baseline_accounting import flush_baseline_accounting
|
||||
|
||||
if await _total_queued_spend_transactions(prisma_client) == 0:
|
||||
await flush_baseline_accounting(prisma_client)
|
||||
return
|
||||
|
||||
logs_to_process: Final = await dequeue_spend_logs(prisma_client, MAX_LOGS_PER_INTERVAL)
|
||||
|
|
@ -7278,6 +7302,8 @@ async def update_spend_logs_job(
|
|||
tool_tracking_err,
|
||||
)
|
||||
|
||||
await flush_baseline_accounting(prisma_client)
|
||||
|
||||
async with prisma_client._autorouter_turn_transactions_lock:
|
||||
autorouter_turns_to_process: Final = prisma_client.autorouter_turn_transactions[:MAX_LOGS_PER_INTERVAL]
|
||||
remaining_autorouter_turns: Final = prisma_client.autorouter_turn_transactions[
|
||||
|
|
@ -7400,7 +7426,9 @@ async def _monitor_spend_logs_queue(
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
else:
|
||||
# Exponential backoff when no logs to process
|
||||
from litellm.proxy.db.baseline_accounting import flush_baseline_accounting
|
||||
|
||||
await flush_baseline_accounting(prisma_client)
|
||||
current_interval = min(current_interval * backoff_multiplier, max_backoff)
|
||||
|
||||
if await _wait_for_spend_log_flush_request(flush_requested, current_interval):
|
||||
|
|
|
|||
48
litellm/repositories/managed_batch_repository.py
Normal file
48
litellm/repositories/managed_batch_repository.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.repositories.table_repositories import PrismaTableRepository
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import models as prisma_models
|
||||
|
||||
|
||||
def _batch_of(blob: object) -> LiteLLMBatch:
|
||||
return LiteLLMBatch.model_validate_json(blob) if isinstance(blob, str) else LiteLLMBatch.model_validate(blob)
|
||||
|
||||
|
||||
class ManagedBatchRepository(PrismaTableRepository["prisma_models.LiteLLM_ManagedObjectTable"]):
|
||||
table_name = "litellm_managedobjecttable"
|
||||
|
||||
async def load_batch(self, unified_batch_id: str) -> LiteLLMBatch | None:
|
||||
row: Final = await self._find_row(unified_batch_id)
|
||||
return None if row is None or not row.file_object else _batch_of(row.file_object)
|
||||
|
||||
async def load_status(self, unified_batch_id: str) -> str | None:
|
||||
row: Final = await self._find_row(unified_batch_id)
|
||||
return row.status if row is not None else None
|
||||
|
||||
async def compare_and_set(
|
||||
self, batch: LiteLLMBatch, unchanged: Mapping[str, object], updated_by: str | None
|
||||
) -> bool:
|
||||
updated_rows: Final = await self.table.update_many(
|
||||
where={"unified_object_id": batch.id, **unchanged}, # mutable-ok: prisma filters are plain dicts
|
||||
data={ # mutable-ok: prisma payloads are plain dicts
|
||||
"file_object": batch.model_dump_json(),
|
||||
"status": batch.status,
|
||||
"updated_by": updated_by,
|
||||
},
|
||||
)
|
||||
return updated_rows > 0
|
||||
|
||||
async def touch(self, unified_batch_id: str, updated_by: str | None) -> None:
|
||||
await self.table.update_many(
|
||||
where={"unified_object_id": unified_batch_id}, # mutable-ok: prisma filters are plain dicts
|
||||
data={"updated_by": updated_by}, # mutable-ok: prisma payloads are plain dicts
|
||||
)
|
||||
|
||||
async def _find_row(self, unified_batch_id: str) -> "prisma_models.LiteLLM_ManagedObjectTable | None":
|
||||
return await self.table.find_first(
|
||||
where={"unified_object_id": unified_batch_id} # mutable-ok: prisma filters are plain dicts
|
||||
)
|
||||
32
litellm/repositories/managed_file_content_repository.py
Normal file
32
litellm/repositories/managed_file_content_repository.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.repositories.table_repositories import PrismaTableRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import models as prisma_models
|
||||
|
||||
|
||||
class ManagedFileContentRepository(PrismaTableRepository["prisma_models.LiteLLM_ManagedFileContentTable"]):
|
||||
table_name = "litellm_managedfilecontenttable"
|
||||
|
||||
async def store(self, content: bytes) -> str:
|
||||
from prisma import Base64
|
||||
|
||||
row: Final = await self.table.create(
|
||||
data={"content": Base64.encode(content)} # mutable-ok: prisma payloads are plain dicts
|
||||
)
|
||||
return row.id
|
||||
|
||||
async def load(self, row_id: str) -> bytes | None:
|
||||
row: Final[prisma_models.LiteLLM_ManagedFileContentTable | None] = await self.table.find_unique(
|
||||
where={"id": row_id} # mutable-ok: prisma filters are plain dicts
|
||||
)
|
||||
return None if row is None else row.content.decode()
|
||||
|
||||
async def delete(self, row_id: str) -> None:
|
||||
from prisma.errors import RecordNotFoundError
|
||||
|
||||
try:
|
||||
await self.table.delete(where={"id": row_id}) # mutable-ok: prisma filters are plain dicts
|
||||
except RecordNotFoundError:
|
||||
return
|
||||
|
|
@ -13723,6 +13723,20 @@ class Router:
|
|||
to the deployment that actually served the request. Every attempt therefore
|
||||
writes or clears, never just writes.
|
||||
"""
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
|
||||
baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None
|
||||
baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None
|
||||
router_name: Final = routing_decision.get("router_model_name") if routing_decision else None
|
||||
Router._stamp_or_clear_metadata_key(
|
||||
request_kwargs=request_kwargs,
|
||||
key="_autorouter_baseline_route",
|
||||
value=(
|
||||
BaselineRouteStamp(router_name, baseline_model, baseline_id)
|
||||
if router_name and baseline_model and baseline_id
|
||||
else None
|
||||
),
|
||||
)
|
||||
Router._stamp_or_clear_metadata_key(
|
||||
request_kwargs=request_kwargs,
|
||||
key="routing_decision",
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ from litellm.types.utils import (
|
|||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
ModelResponse,
|
||||
RoutingDecisionCause,
|
||||
StandardLoggingHeuristicV2Forecast,
|
||||
StandardLoggingRoutingDecision,
|
||||
StandardLoggingRoutingDecisionTierBoundaries,
|
||||
)
|
||||
|
|
@ -1043,6 +1044,7 @@ class ClassificationOutcome(NamedTuple):
|
|||
capability_forecast: CapabilityClassifierForecast | None = None
|
||||
llm_v2_forecast: LLMV2Decision | None = None
|
||||
jev_verdict: JevVerdict | None = None
|
||||
heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None
|
||||
|
||||
|
||||
def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome:
|
||||
|
|
@ -1075,6 +1077,8 @@ def _with_classifier_forecast(
|
|||
decision: StandardLoggingRoutingDecision, outcome: ClassificationOutcome
|
||||
) -> StandardLoggingRoutingDecision:
|
||||
"""Attach validated forecasts and their applied policy to the routing decision."""
|
||||
if outcome.heuristic_v2_forecast is not None:
|
||||
return {**decision, "heuristic_v2_forecast": outcome.heuristic_v2_forecast}
|
||||
if outcome.jev_verdict is not None:
|
||||
forecasted_decision: Final[StandardLoggingRoutingDecision] = {
|
||||
**decision,
|
||||
|
|
@ -1772,6 +1776,7 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing: bool = True,
|
||||
tier_litellm_params: Mapping[str, object] | None = None,
|
||||
context_escalation_original_tier: ComplexityTier | str | None = None,
|
||||
heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None,
|
||||
) -> StandardLoggingRoutingDecision:
|
||||
"""Assemble the per-request provenance record for this router's decision.
|
||||
|
||||
|
|
@ -1831,7 +1836,9 @@ class ComplexityRouter(CustomLogger):
|
|||
masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params)
|
||||
if isinstance(masked_tier_litellm_params, Mapping):
|
||||
decision["tier_litellm_params"] = masked_tier_litellm_params
|
||||
return decision
|
||||
return (
|
||||
decision if heuristic_v2_forecast is None else {**decision, "heuristic_v2_forecast": heuristic_v2_forecast}
|
||||
)
|
||||
|
||||
async def aclassify(
|
||||
self,
|
||||
|
|
@ -1888,6 +1895,15 @@ class ComplexityRouter(CustomLogger):
|
|||
score=None,
|
||||
signals=(f"request-type:{request_type.value}", *probability_signals),
|
||||
cause="heuristic_v2",
|
||||
heuristic_v2_forecast=StandardLoggingHeuristicV2Forecast(
|
||||
probabilities={
|
||||
candidate.value: prediction.probabilities[index]
|
||||
for index, candidate in enumerate(TIER_SEVERITY_ORDER, start=1)
|
||||
},
|
||||
threshold=predictor.routing_threshold,
|
||||
predicted_tier=tier.value,
|
||||
request_type=request_type.value,
|
||||
),
|
||||
)
|
||||
|
||||
async def _classify_heuristic_first(
|
||||
|
|
@ -3553,6 +3569,7 @@ class ComplexityRouter(CustomLogger):
|
|||
context_escalation_original_tier=(
|
||||
decision.get("context_escalation_original_tier") if decision is not None else None
|
||||
),
|
||||
heuristic_v2_forecast=decision.get("heuristic_v2_forecast") if decision is not None else None,
|
||||
)
|
||||
from litellm.types.router import PreRoutingHookResponse as HookResponse
|
||||
|
||||
|
|
@ -3732,6 +3749,7 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing=bool(decision.get("conversation_continuing", True)),
|
||||
tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model),
|
||||
context_escalation_original_tier=decision.get("context_escalation_original_tier"),
|
||||
heuristic_v2_forecast=decision.get("heuristic_v2_forecast"),
|
||||
)
|
||||
return response.model_copy(
|
||||
update={ # mutable-ok: model_copy types update as a plain dict
|
||||
|
|
@ -3776,6 +3794,7 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing=bool(decision.get("conversation_continuing", True)),
|
||||
tier_litellm_params=self._litellm_params_for_model(None, default_model),
|
||||
context_escalation_original_tier=decision.get("context_escalation_original_tier"),
|
||||
heuristic_v2_forecast=decision.get("heuristic_v2_forecast"),
|
||||
)
|
||||
return response.model_copy(
|
||||
update={ # mutable-ok: model_copy types update as a plain dict
|
||||
|
|
|
|||
100
litellm/router_strategy/complexity_router/fuse_presets.json
Normal file
100
litellm/router_strategy/complexity_router/fuse_presets.json
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"version": "2026-09-17-v1",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-6-astra-v1",
|
||||
"label": "GPT-6 Astra",
|
||||
"model": "gpt-6-astra",
|
||||
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol-v1",
|
||||
"label": "GPT-5.6 Sol",
|
||||
"model": "gpt-5.6-sol",
|
||||
"text": "OpenAI model for complex professional work, supporting reasoning and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-sol"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-luna-v1",
|
||||
"label": "GPT-5.6 Luna",
|
||||
"model": "gpt-5.6-luna",
|
||||
"text": "OpenAI model for high-volume workloads, supporting reasoning and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-luna"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-terra-v1",
|
||||
"label": "GPT-5.6 Terra",
|
||||
"model": "gpt-5.6-terra",
|
||||
"text": "OpenAI general-purpose model supporting reasoning, text and image input, and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-terra"]
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5-v1",
|
||||
"label": "Claude Haiku 4.5",
|
||||
"model": "claude-haiku-4-5",
|
||||
"text": "Anthropic latency-focused model supporting text and image input, tool use, and extended thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/haiku-4-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-sonnet-5-v1",
|
||||
"label": "Claude Sonnet 5",
|
||||
"model": "claude-sonnet-5",
|
||||
"text": "Anthropic model balancing speed and capability, with adaptive thinking and tool use",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/sonnet-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-opus-5-v1",
|
||||
"label": "Claude Opus 5",
|
||||
"model": "claude-opus-5",
|
||||
"text": "Anthropic model for complex agentic coding and enterprise work, with adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/opus-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-fable-5-v1",
|
||||
"label": "Claude Fable 5",
|
||||
"model": "claude-fable-5",
|
||||
"text": "Anthropic model for demanding reasoning and long-running agent tasks, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5/introducing-claude-fable-5-and-claude-mythos-5"]
|
||||
},
|
||||
{
|
||||
"id": "claude-fable-5-1-v1",
|
||||
"label": "Claude Fable 5.1",
|
||||
"model": "claude-fable-5-1",
|
||||
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
|
||||
}
|
||||
],
|
||||
"harnesses": [
|
||||
{
|
||||
"id": "unspecified-v1",
|
||||
"label": "Unspecified runtime",
|
||||
"text": "Agent runtime is unspecified. Assess the task using the supplied context without assuming repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://code.claude.com/docs/en/how-claude-code-works", "https://mini-swe-agent.com/latest/faq/"]
|
||||
},
|
||||
{
|
||||
"id": "claude-code-v1",
|
||||
"label": "Claude Code",
|
||||
"text": "Claude Code supplies an agent loop with context management and configured tools. Available actions depend on the session's tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://code.claude.com/docs/en/how-claude-code-works"]
|
||||
},
|
||||
{
|
||||
"id": "codex-cli-v1",
|
||||
"label": "Codex CLI",
|
||||
"text": "Codex CLI supplies a terminal-based coding agent. File operations, command execution, and integrations depend on the session's tools, permissions, and sandbox. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://learn.chatgpt.com/docs/codex/cli", "https://learn.chatgpt.com/codex/permissions"]
|
||||
},
|
||||
{
|
||||
"id": "opencode-v1",
|
||||
"label": "OpenCode",
|
||||
"text": "OpenCode supplies a configurable agent runtime. Available actions depend on the selected agent, tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://opencode.ai/docs/agents/"]
|
||||
},
|
||||
{
|
||||
"id": "mini-swe-agent-v1",
|
||||
"label": "mini-SWE-agent",
|
||||
"text": "The standard mini-SWE-agent setup uses a bash-only action interface and separate command executions. Available commands and resources depend on its configured environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://mini-swe-agent.com/latest/faq/"]
|
||||
}
|
||||
]
|
||||
}
|
||||
52
litellm/router_strategy/complexity_router/fuse_presets.py
Normal file
52
litellm/router_strategy/complexity_router/fuse_presets.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from functools import lru_cache
|
||||
from importlib.resources import files
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
||||
|
||||
ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)]
|
||||
|
||||
|
||||
class FuseModelPreset(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
id: str
|
||||
label: str
|
||||
text: ProfileText
|
||||
sources: tuple[str, ...] = Field(min_length=1)
|
||||
model: str
|
||||
|
||||
|
||||
class FuseHarnessPreset(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
id: str
|
||||
label: str
|
||||
text: ProfileText
|
||||
sources: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class FusePresetCatalog(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
version: str
|
||||
models: tuple[FuseModelPreset, ...]
|
||||
harnesses: tuple[FuseHarnessPreset, ...]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_fuse_presets() -> FusePresetCatalog:
|
||||
return FusePresetCatalog.model_validate_json(
|
||||
files(__package__).joinpath("fuse_presets.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
def resolve_fuse_profile(text: str | None, preset_id: str | None, kind: Literal["model", "harness"]) -> str | None:
|
||||
if preset_id is None:
|
||||
return text
|
||||
catalog: Final = get_fuse_presets()
|
||||
presets: Final = catalog.models if kind == "model" else catalog.harnesses
|
||||
preset: Final = next((entry for entry in presets if entry.id == preset_id), None)
|
||||
if preset is None:
|
||||
return None
|
||||
return text if text is not None else preset.text
|
||||
|
|
@ -7,15 +7,15 @@ from dataclasses import dataclass
|
|||
from sys import float_info
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter, model_validator
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # legacy output validated below
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import ProfileText, resolve_fuse_profile
|
||||
|
||||
ShortText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=512)]
|
||||
ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)]
|
||||
|
||||
|
||||
class _SolverProfile(TypedDict):
|
||||
|
|
@ -139,20 +139,41 @@ class LLMV2Config(BaseModel):
|
|||
|
||||
efficient_tier: str = "SIMPLE"
|
||||
capable_tier: str = "REASONING"
|
||||
efficient_profile: ProfileText
|
||||
capable_profile: ProfileText
|
||||
harness: ProfileText
|
||||
efficient_profile: ProfileText | None = None
|
||||
capable_profile: ProfileText | None = None
|
||||
harness: ProfileText | None = None
|
||||
efficient_profile_preset: str | None = None
|
||||
capable_profile_preset: str | None = None
|
||||
harness_preset: str | None = None
|
||||
max_quality_gap: float = Field(ge=0.0, le=1.0, description="Maximum estimated success loss allowed for efficient.")
|
||||
max_output_tokens: int = Field(default=1024, ge=1)
|
||||
response_format: Literal["json_schema", "json_object"] = "json_schema"
|
||||
calibration: LLMV2Calibration | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_profiles(self) -> LLMV2Config:
|
||||
self._profile_texts()
|
||||
return self
|
||||
|
||||
def _profile_texts(self) -> tuple[str, str, str]:
|
||||
efficient: Final = resolve_fuse_profile(self.efficient_profile, self.efficient_profile_preset, "model")
|
||||
capable: Final = resolve_fuse_profile(self.capable_profile, self.capable_profile_preset, "model")
|
||||
harness: Final = resolve_fuse_profile(self.harness, self.harness_preset, "harness")
|
||||
if efficient is None:
|
||||
raise ValueError("efficient_profile requires text or a known efficient_profile_preset")
|
||||
if capable is None:
|
||||
raise ValueError("capable_profile requires text or a known capable_profile_preset")
|
||||
if harness is None:
|
||||
raise ValueError("harness requires text or a known harness_preset")
|
||||
return efficient, capable, harness
|
||||
|
||||
def system_prompt(self, efficient_model: str, capable_model: str) -> str:
|
||||
efficient, capable, harness = self._profile_texts()
|
||||
profiles: Final[_SolverProfiles] = {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": self.harness,
|
||||
"efficient": {"model": efficient_model, "profile": self.efficient_profile},
|
||||
"capable": {"model": capable_model, "profile": self.capable_profile},
|
||||
"harness": harness,
|
||||
"efficient": {"model": efficient_model, "profile": efficient},
|
||||
"capable": {"model": capable_model, "profile": capable},
|
||||
}
|
||||
schema: Final = (
|
||||
"\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema())
|
||||
|
|
|
|||
|
|
@ -512,6 +512,7 @@ class CreateBatchRequest(TypedDict, total=False):
|
|||
|
||||
class LiteLLMBatchCreateRequest(CreateBatchRequest, total=False):
|
||||
model: str
|
||||
disable_fallbacks: ReadOnly[bool]
|
||||
|
||||
|
||||
class RetrieveBatchRequest(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -199,13 +199,20 @@ class AutoRouterBenchmarkTotals(BaseModel):
|
|||
description="Recorded LLM classifier cost already included in spend; null when any session turns predate "
|
||||
"subtotal recording, and zero for an empty window"
|
||||
)
|
||||
saved_spend: float = Field(
|
||||
description="Signed dollars saved versus each router's savings baseline (derived from its hardest "
|
||||
"tier, or the configured override), from the same per-request savings record the usage tab reads"
|
||||
savings_estimated_turns: int = Field(
|
||||
description="Turns covered by the current savings estimator; legacy estimates are excluded"
|
||||
)
|
||||
savings_estimated_actual_spend: float = Field(
|
||||
description="Actual spend, including classifier cost, for covered turns only"
|
||||
)
|
||||
saved_spend: float | None = Field(
|
||||
description="Signed savings for covered turns only; null when traffic has no current estimates"
|
||||
)
|
||||
baseline_spend: float | None = Field(description="Estimated single-model cost for covered turns only")
|
||||
saved_pct: float | None = Field(description="Covered savings over covered baseline spend, as a percentage")
|
||||
saved_per_session: float | None = Field(
|
||||
description="Average session savings; unavailable unless every turn is covered"
|
||||
)
|
||||
baseline_spend: float = Field(description="spend plus saved_spend: the estimated single-model cost")
|
||||
saved_pct: float = Field(description="saved_spend over baseline_spend, as a percentage")
|
||||
saved_per_session: float
|
||||
cache: AutoRouterCacheStats
|
||||
|
||||
|
||||
|
|
@ -236,16 +243,27 @@ class AutoRouterSessionResponse(BaseModel):
|
|||
turns: int = Field(description="Auto-routed turns the rollup has recorded for this session so far")
|
||||
last_model: str = Field(description="The deployment model the most recent turn was routed to")
|
||||
spend: float = Field(description="What the session's routed traffic actually cost, classifier calls included")
|
||||
saved_spend: float = Field(description="Estimated savings against the baseline, net of classifier cost")
|
||||
baseline_spend: float = Field(description="spend plus saved_spend: the estimated single-model cost")
|
||||
savings_estimated_turns: int = Field(
|
||||
description="Turns covered by the current savings estimator; legacy estimates are excluded"
|
||||
)
|
||||
savings_estimated_actual_spend: float = Field(
|
||||
description="Actual spend, including classifier cost, for covered turns only"
|
||||
)
|
||||
saved_spend: float | None = Field(description="Estimated savings for covered turns only, net of classifier cost")
|
||||
baseline_spend: float | None = Field(
|
||||
description="Estimated single-model cost; unavailable unless every turn is covered"
|
||||
)
|
||||
savings_estimated_baseline_spend: float | None = Field(
|
||||
description="Estimated single-model cost for covered turns only"
|
||||
)
|
||||
baseline_model: str | None = Field(
|
||||
description="The savings baseline most of this session's turns were priced against, recorded turn by "
|
||||
description="The savings baseline most covered turns were priced against, recorded turn by "
|
||||
"turn, so it still names the counterfactual after the router is reconfigured or removed. None when no "
|
||||
"turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, "
|
||||
"which derive no baseline and so report no savings"
|
||||
)
|
||||
baseline_models: Mapping[str, int] = Field(
|
||||
description="Turns priced against each baseline model; more than one entry means the router's "
|
||||
description="Covered turns priced against each baseline model; more than one entry means the router's "
|
||||
"baseline changed mid-session and baseline_spend mixes both"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1057,6 +1057,13 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]):
|
|||
strategy: _PreRoutingStrategyT_co
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BaselineRouteStamp:
|
||||
router_name: str
|
||||
baseline_model: str
|
||||
baseline_deployment_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConsumedRequestTagsStamp:
|
||||
"""The model group a tagged router rewrote to, plus the request tags spent selecting it."""
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_assistant_prefill: bool | None
|
||||
supports_prompt_caching: bool | None
|
||||
supports_prompt_cache_breakpoint: ReadOnly[bool | None]
|
||||
supports_thinking_cache_preservation: ReadOnly[bool | None]
|
||||
supports_computer_use: bool | None
|
||||
supports_audio_input: bool | None
|
||||
supports_embedding_image_input: bool | None
|
||||
|
|
@ -2974,6 +2975,13 @@ LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN: Final[InternalCallOrigin] = "llm_as_a_judg
|
|||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
|
||||
|
||||
|
||||
class StandardLoggingHeuristicV2Forecast(TypedDict):
|
||||
probabilities: ReadOnly[Mapping[str, float]]
|
||||
threshold: ReadOnly[float]
|
||||
predicted_tier: ReadOnly[str]
|
||||
request_type: ReadOnly[str]
|
||||
|
||||
|
||||
class StandardLoggingRoutingDecision(TypedDict, total=False):
|
||||
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
|
||||
|
||||
|
|
@ -2992,6 +3000,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
|
|||
classifier_cost: float
|
||||
classifier_probabilities: ReadOnly[Mapping[str, float]]
|
||||
classifier_confidence: ReadOnly[float]
|
||||
heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast]
|
||||
classifier_crux: str # writable-ok: added only when a capability verdict is available
|
||||
classifier_primary_rule: str # writable-ok: added only when a capability verdict is available
|
||||
classifier_capability_boundary: str # writable-ok: added only when a capability verdict is available
|
||||
|
|
@ -3037,6 +3046,7 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"classifier_cost",
|
||||
"classifier_probabilities",
|
||||
"classifier_confidence",
|
||||
"heuristic_v2_forecast",
|
||||
"classifier_primary_rule",
|
||||
"classifier_capability_boundary",
|
||||
"classifier_p_solve",
|
||||
|
|
@ -3450,7 +3460,9 @@ class StandardLoggingPayload(ClassifierAudit):
|
|||
stream: bool | None
|
||||
response_cost: float
|
||||
cost_breakdown: CostBreakdown | None # Detailed cost breakdown
|
||||
autorouter_savings: ReadOnly[float | None] # None = not an auto-routed caller request; 0.0 is a real figure
|
||||
autorouter_savings: ReadOnly[float | None]
|
||||
autorouter_savings_estimate: ReadOnly[Mapping[str, JsonValue] | None]
|
||||
autorouter_baseline_observation: ReadOnly[str | None]
|
||||
response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None
|
||||
status: StandardLoggingPayloadStatus
|
||||
status_fields: StandardLoggingPayloadStatusFields
|
||||
|
|
@ -4178,6 +4190,8 @@ FILE_CONTENT_STREAMING_PROVIDERS: Final[frozenset[str]] = frozenset(
|
|||
{*OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, LlmProviders.VERTEX_AI.value}
|
||||
)
|
||||
|
||||
LITELLM_EXECUTED_BATCH_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.HOSTED_VLLM.value})
|
||||
|
||||
ListBatchesSupportedProvider = Literal["openai", "azure", "hosted_vllm", "litellm_proxy", "vertex_ai"]
|
||||
|
||||
LIST_BATCHES_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset(get_args(ListBatchesSupportedProvider))
|
||||
|
|
|
|||
|
|
@ -1881,6 +1881,7 @@ def client(original_function):
|
|||
# Type assertion: logging_obj is guaranteed to be non-None after function_setup
|
||||
assert logging_obj is not None, "logging_obj should not be None after function_setup"
|
||||
|
||||
kwargs["litellm_logging_obj"] = logging_obj
|
||||
modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type)
|
||||
if modified_kwargs is not None:
|
||||
kwargs = modified_kwargs
|
||||
|
|
@ -2848,6 +2849,14 @@ def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None
|
|||
)
|
||||
|
||||
|
||||
def supports_thinking_cache_preservation(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
return _supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
key="supports_thinking_cache_preservation",
|
||||
)
|
||||
|
||||
|
||||
def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
"""
|
||||
Check if the given model supports computer use and return a boolean value.
|
||||
|
|
@ -5822,6 +5831,7 @@ def _get_model_info_helper(
|
|||
supports_assistant_prefill=None,
|
||||
supports_prompt_caching=None,
|
||||
supports_prompt_cache_breakpoint=None,
|
||||
supports_thinking_cache_preservation=None,
|
||||
supports_computer_use=None,
|
||||
supports_pdf_input=None,
|
||||
)
|
||||
|
|
@ -6094,6 +6104,7 @@ def _get_model_info_helper(
|
|||
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
|
||||
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
|
||||
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
|
||||
supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None),
|
||||
supports_audio_input=_model_info.get("supports_audio_input", None),
|
||||
supports_audio_output=_model_info.get("supports_audio_output", None),
|
||||
supports_pdf_input=_model_info.get("supports_pdf_input", None),
|
||||
|
|
|
|||
|
|
@ -14510,6 +14510,7 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
|
|
@ -14547,6 +14548,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14698,6 +14700,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14727,6 +14730,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14759,6 +14763,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14796,6 +14801,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14831,6 +14837,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14869,6 +14876,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14986,6 +14994,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -15027,6 +15036,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_thinking_cache_preservation": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
|
|||
|
|
@ -911,6 +911,9 @@
|
|||
"supports_system_messages": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_thinking_cache_preservation": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_tool_choice": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -302,6 +302,7 @@ editable-profile = "dev"
|
|||
include = [
|
||||
"litellm/proxy/_experimental/out/**",
|
||||
"litellm/router_strategy/complexity_router/artifacts/*.json",
|
||||
"litellm/router_strategy/complexity_router/fuse_presets.json",
|
||||
"litellm/proxy/client/cli/commands/codex_base_instructions.md",
|
||||
]
|
||||
exclude = [
|
||||
|
|
|
|||
|
|
@ -1107,6 +1107,12 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
|
|||
@@index([team_id, created_at(sort: Desc)])
|
||||
}
|
||||
|
||||
model LiteLLM_ManagedFileContentTable {
|
||||
id String @id @default(uuid())
|
||||
content Bytes
|
||||
created_at DateTime @default(now())
|
||||
}
|
||||
|
||||
model LiteLLM_ManagedVectorStoreTable {
|
||||
id String @id @default(uuid())
|
||||
unified_resource_id String @unique // The base64 encoded unified vector store ID
|
||||
|
|
@ -1545,6 +1551,36 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterBaselineComparison {
|
||||
scope String @id
|
||||
api_key String
|
||||
session_id String
|
||||
router_name String
|
||||
initial_equivalent Boolean
|
||||
revision BigInt @default(0)
|
||||
published_revision BigInt @default(0)
|
||||
history String?
|
||||
attempted_at DateTime?
|
||||
retired Boolean @default(false)
|
||||
updated_at DateTime @default(now())
|
||||
|
||||
@@index([api_key, session_id, router_name], map: "idx_autorouter_baseline_scope")
|
||||
@@index([updated_at], map: "idx_autorouter_baseline_updated")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterBaselineObservation {
|
||||
request_id String @id
|
||||
scope String
|
||||
started_at Float
|
||||
revision BigInt
|
||||
data String
|
||||
publication String?
|
||||
conflicted Boolean @default(false)
|
||||
|
||||
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")
|
||||
@@index([scope, revision, started_at], map: "idx_autorouter_baseline_event_revision")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterSession {
|
||||
api_key String
|
||||
session_id String
|
||||
|
|
@ -1571,6 +1607,10 @@ model LiteLLM_AutoRouterSession {
|
|||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
savings_estimated_baseline_models Json @default("{}")
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
|
|
|
|||
|
|
@ -1160,62 +1160,151 @@ def _vllm_params(api_base: str, api_key: str | None, model_id: str) -> LiteLLMPa
|
|||
)
|
||||
|
||||
|
||||
class TestHostedVllmBatch:
|
||||
"""hosted_vllm file upload + batch create (OpenAI-compatible path, LIT-3266).
|
||||
HOSTED_VLLM_DEFAULT_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
HOSTED_VLLM_BAD_LINE_CUSTOM_ID = "req-bad"
|
||||
|
||||
hosted_vllm is in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, so /v1/files
|
||||
and /v1/batches route through the OpenAI handler against the deployment's
|
||||
api_base. Skipped for now: it needs a live vLLM (or OpenAI-compatible) server
|
||||
exposing the files/batches APIs (HOSTED_VLLM_API_BASE), which the e2e
|
||||
environment does not currently provision.
|
||||
|
||||
def _hosted_vllm_deployment(client: BatchClient, resources: ResourceManager) -> str:
|
||||
api_base = os.environ.get("HOSTED_VLLM_API_BASE")
|
||||
if api_base is None:
|
||||
pytest.skip("set HOSTED_VLLM_API_BASE (the live vLLM server this deployment targets)")
|
||||
api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None
|
||||
model_id = (os.environ.get("HOSTED_VLLM_MODEL") or HOSTED_VLLM_DEFAULT_MODEL).strip()
|
||||
proxy_name = batch_model_name("hosted-vllm-batch")
|
||||
model_row_id = client.create_model(proxy_name, _vllm_params(api_base, api_key, model_id))
|
||||
resources.defer(lambda: client.delete_model(model_row_id))
|
||||
return proxy_name
|
||||
|
||||
|
||||
def _upload_hosted_vllm_input(
|
||||
client: BatchClient, content: bytes, *, proxy_name: str, key: str, upload_route: str
|
||||
) -> Result[FileObject]:
|
||||
if upload_route == "model_query":
|
||||
return client.upload_file(content=content, form=FileUploadForm(purpose="batch"), model=proxy_name, key=key)
|
||||
return client.upload_file(
|
||||
content=content, form=FileUploadForm(purpose="batch", target_model_names=proxy_name), key=key
|
||||
)
|
||||
|
||||
|
||||
def _jsonl_with_a_failing_line(model: str) -> bytes:
|
||||
bad_line = {
|
||||
"custom_id": HOSTED_VLLM_BAD_LINE_CUSTOM_ID,
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {"model": model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": -1},
|
||||
}
|
||||
return render_jsonl(model) + (json.dumps(bad_line) + "\n").encode()
|
||||
|
||||
|
||||
def _download_managed_file(client: BatchClient, file_id: str, *, key: str) -> list[str]:
|
||||
downloaded = client.proxy.transport.download(
|
||||
f"/v1/files/{file_id}/content", headers=client.proxy.transport.bearer(key)
|
||||
)
|
||||
assert downloaded.status_code == 200, (
|
||||
f"file content must be 200, got {downloaded.status_code}: {downloaded.body[:300]}"
|
||||
)
|
||||
return downloaded.body.strip().splitlines()
|
||||
|
||||
|
||||
class TestHostedVllmBatch:
|
||||
"""hosted_vllm file upload + batch execution (LIT-5739).
|
||||
|
||||
vLLM implements neither /v1/files nor /v1/batches, so LiteLLM keeps the batch
|
||||
input in its own database, runs every line through the deployment's
|
||||
/v1/chat/completions itself, and serves the batch plus its output and error
|
||||
files from that database under the creating key. Needs a live vLLM server
|
||||
(HOSTED_VLLM_API_BASE), which the default e2e stack does not provision, so
|
||||
the cases skip without it.
|
||||
"""
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="hosted_vllm batch/files needs a live vLLM server (HOSTED_VLLM_API_BASE) "
|
||||
"not provisioned in the e2e environment; re-enable when available (LIT-3266)"
|
||||
)
|
||||
@pytest.mark.parametrize("upload_route", ["target_model_names", "model_query"])
|
||||
@pytest.mark.covers(
|
||||
"llm.batches.hosted_vllm.basic.nonstream.works",
|
||||
"llm.files.hosted_vllm.upload.nonstream.works",
|
||||
exercised_on=["batches", "files"],
|
||||
)
|
||||
def test_unified_file_and_batch_create(
|
||||
self, client: BatchClient, resources: ResourceManager
|
||||
def test_batch_runs_to_completion_with_a_downloadable_output(
|
||||
self, client: BatchClient, resources: ResourceManager, upload_route: str
|
||||
) -> None:
|
||||
api_base = os.environ["HOSTED_VLLM_API_BASE"]
|
||||
api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None
|
||||
model_id = (
|
||||
os.environ.get("HOSTED_VLLM_MODEL") or "meta-llama/Llama-3.2-3B-Instruct"
|
||||
).strip()
|
||||
proxy_name = batch_model_name("hosted-vllm-batch")
|
||||
|
||||
model_row_id = client.create_model(
|
||||
proxy_name, _vllm_params(api_base, api_key, model_id)
|
||||
)
|
||||
resources.defer(lambda: client.delete_model(model_row_id))
|
||||
proxy_name = _hosted_vllm_deployment(client, resources)
|
||||
key = resources.key()
|
||||
|
||||
file = unwrap(
|
||||
client.upload_file(
|
||||
content=render_jsonl(model_id),
|
||||
form=FileUploadForm(purpose="batch", target_model_names=proxy_name),
|
||||
key=key,
|
||||
_upload_hosted_vllm_input(
|
||||
client, render_jsonl(proxy_name), proxy_name=proxy_name, key=key, upload_route=upload_route
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: cleanup_file(client, file.id, key=key))
|
||||
assert_file_object(file, provider="hosted_vllm")
|
||||
assert is_managed_id(file.id), f"hosted_vllm batch input must stay in LiteLLM, got file id {file.id!r}"
|
||||
|
||||
created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key)
|
||||
require_successful_call(created)
|
||||
batch = BatchObject.model_validate_json(created.body)
|
||||
resources.defer(lambda: cleanup_batch(client, batch.id, key=key))
|
||||
|
||||
assert batch.id, f"hosted_vllm create returned no batch id: {created.body[:200]}"
|
||||
assert batch.status in CREATED_BATCH_STATUSES, (
|
||||
f"hosted_vllm batch has non-transitional status {batch.status!r}"
|
||||
)
|
||||
resources.defer(lambda: cleanup_batch(client, batch.id, key=key, delete_output_files=True))
|
||||
assert is_managed_id(batch.id), f"hosted_vllm batch must be LiteLLM-managed, got {batch.id!r}"
|
||||
assert batch.status in CREATED_BATCH_STATUSES, f"hosted_vllm batch has non-transitional status {batch.status!r}"
|
||||
assert_batch_object(batch)
|
||||
|
||||
finished = _poll_until_terminal(client, batch.id, key)
|
||||
assert finished.status == "completed", f"hosted_vllm batch ended {finished.status!r}: {finished.errors!r}"
|
||||
assert finished.output_file_id, "completed hosted_vllm batch has no output_file_id"
|
||||
assert finished.error_file_id is None, f"all lines succeeded but error_file_id={finished.error_file_id!r}"
|
||||
|
||||
output_lines = _download_managed_file(client, finished.output_file_id, key=key)
|
||||
assert len(output_lines) == 1, f"one input line must yield one output line, got {output_lines!r}"
|
||||
first_line = BatchOutputLine.model_validate_json(output_lines[0])
|
||||
assert first_line.custom_id == "req-1", f"output line lost its custom_id: {output_lines[0][:300]}"
|
||||
assert first_line.response.status_code == 200, f"batch output line reports failure: {output_lines[0][:400]}"
|
||||
assert first_line.response.body is not None and first_line.response.body.choices, (
|
||||
"batch output line has no choices"
|
||||
)
|
||||
|
||||
rows = client.proxy.poll_logs_for_key(
|
||||
key, predicate=lambda found: any(row.call_type == "acompletion" for row in found)
|
||||
)
|
||||
line_rows = [row for row in rows if row.call_type == "acompletion"]
|
||||
assert line_rows, f"the batch line's chat call was not logged under the creating key: {rows!r}"
|
||||
assert all(row.custom_llm_provider == "hosted_vllm" for row in line_rows), (
|
||||
f"batch line rows must be attributed to hosted_vllm: {line_rows!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.batches.hosted_vllm.basic.nonstream.works", exercised_on=["batches", "files"])
|
||||
def test_failing_line_lands_in_the_error_file_not_the_batch_status(
|
||||
self, client: BatchClient, resources: ResourceManager
|
||||
) -> None:
|
||||
proxy_name = _hosted_vllm_deployment(client, resources)
|
||||
key = resources.key()
|
||||
|
||||
file = unwrap(
|
||||
_upload_hosted_vllm_input(
|
||||
client,
|
||||
_jsonl_with_a_failing_line(proxy_name),
|
||||
proxy_name=proxy_name,
|
||||
key=key,
|
||||
upload_route="target_model_names",
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: cleanup_file(client, file.id, key=key))
|
||||
|
||||
created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key)
|
||||
require_successful_call(created)
|
||||
batch = BatchObject.model_validate_json(created.body)
|
||||
resources.defer(lambda: cleanup_batch(client, batch.id, key=key, delete_output_files=True))
|
||||
|
||||
finished = _poll_until_terminal(client, batch.id, key)
|
||||
assert finished.status == "completed", f"a failing line must not fail the batch, got {finished.status!r}"
|
||||
assert finished.output_file_id, "the good line must still produce an output file"
|
||||
assert finished.error_file_id, "the failing line must produce an error file"
|
||||
|
||||
output_lines = _download_managed_file(client, finished.output_file_id, key=key)
|
||||
error_lines = _download_managed_file(client, finished.error_file_id, key=key)
|
||||
assert [BatchOutputLine.model_validate_json(line).custom_id for line in output_lines] == ["req-1"]
|
||||
assert len(error_lines) == 1, f"one failing line must yield one error line, got {error_lines!r}"
|
||||
error_line = BatchOutputLine.model_validate_json(error_lines[0])
|
||||
assert error_line.custom_id == HOSTED_VLLM_BAD_LINE_CUSTOM_ID
|
||||
assert error_line.response.status_code == 400, f"error line must carry the provider's 4xx: {error_lines[0][:400]}"
|
||||
|
||||
|
||||
BATCH_TERMINAL_STATUSES = frozenset({"completed", "failed", "expired", "cancelled"})
|
||||
FAILED_BATCH_POLL_SECONDS = 120.0
|
||||
|
|
@ -1443,6 +1532,7 @@ class BatchOutputResponse(BaseModel):
|
|||
|
||||
|
||||
class BatchOutputLine(BaseModel):
|
||||
custom_id: str | None = None
|
||||
response: BatchOutputResponse
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from prisma import Prisma
|
||||
|
||||
from litellm.proxy.db.autorouter_session_rollup import (
|
||||
AUTOROUTER_BENCHMARKS_SQL,
|
||||
|
|
@ -43,6 +44,7 @@ async def _turn(
|
|||
classifier_cost: float = 0.0,
|
||||
tier: "str | None" = None,
|
||||
baseline: "str | None" = None,
|
||||
estimated: bool = True,
|
||||
) -> None:
|
||||
touched: Final = 1 if (hit or ttl is not None or not covered) else 0
|
||||
await db.execute_raw(
|
||||
|
|
@ -63,6 +65,9 @@ async def _turn(
|
|||
touched,
|
||||
tier,
|
||||
baseline,
|
||||
int(estimated),
|
||||
spend if estimated else 0.0,
|
||||
saved if estimated else 0.0,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -208,6 +213,9 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers
|
|||
assert row["saved_spend"] == pytest.approx(0.02 * len(writers))
|
||||
assert row["classifier_cost"] == pytest.approx(0.004 * sum(writers))
|
||||
assert row["classifier_cost_recorded_turns"] == sum(writers)
|
||||
assert row["savings_estimated_turns"] == sum(writers)
|
||||
assert row["savings_estimated_actual_spend"] == pytest.approx(0.01 * sum(writers))
|
||||
assert row["savings_estimated_saved_spend"] == pytest.approx(0.02 * sum(writers))
|
||||
groups: Final = await db.query_raw(
|
||||
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
|
||||
)
|
||||
|
|
@ -217,6 +225,32 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers
|
|||
assert groups[0]["turns"] == len(writers)
|
||||
assert groups[0]["spend"] == row["spend"]
|
||||
assert groups[0]["saved_spend"] == row["saved_spend"]
|
||||
assert groups[0]["savings_estimated_turns"] == sum(writers)
|
||||
assert groups[0]["savings_estimated_actual_spend"] == row["savings_estimated_actual_spend"]
|
||||
assert groups[0]["savings_estimated_saved_spend"] == row["savings_estimated_saved_spend"]
|
||||
|
||||
|
||||
async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_the_estimated_cohort(db: Prisma) -> None:
|
||||
key: Final = f"k-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, spend=0.25, saved=-0.05, baseline="opus")
|
||||
await _turn(
|
||||
db, key, "B", T0 + timedelta(seconds=1), spend=0.7, saved=0, baseline="sonnet", estimated=False
|
||||
)
|
||||
await _legacy_turn(db, key, T0 + timedelta(seconds=2))
|
||||
|
||||
row: Final = await _row(db, key)
|
||||
assert row["saved_spend"] == pytest.approx(-0.03)
|
||||
assert row["savings_estimated_baseline_models"] == {"opus": 1}
|
||||
groups: Final = await db.query_raw(
|
||||
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
|
||||
)
|
||||
assert len(groups) == 1
|
||||
for actual in (row, groups[0]):
|
||||
assert actual["turns"] == 3
|
||||
assert actual["spend"] == pytest.approx(0.96)
|
||||
assert actual["savings_estimated_turns"] == 1
|
||||
assert actual["savings_estimated_actual_spend"] == pytest.approx(0.25)
|
||||
assert actual["savings_estimated_saved_spend"] == pytest.approx(-0.05)
|
||||
|
||||
|
||||
async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db):
|
||||
|
|
|
|||
263
tests/proxy_behavior/spend/test_baseline_accounting.py
Normal file
263
tests/proxy_behavior/spend/test_baseline_accounting.py
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from prisma import Prisma
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.prompt_cache_prediction import CountedBreakpoint, CountedPromptCachePlan
|
||||
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
|
||||
from litellm.proxy.db.baseline_accounting import (
|
||||
BaselineAccountingRecord,
|
||||
BaselineAccountingStore,
|
||||
DailyBaselineAttribution,
|
||||
DailyBaselineTarget,
|
||||
)
|
||||
from litellm.proxy.db.create_views import SupportsRawQueries
|
||||
from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation
|
||||
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _transaction(db: Prisma, *, before_commit: bool = False, after_commit: bool = False) -> AsyncIterator[SupportsRawQueries]:
|
||||
async with db.tx() as tx:
|
||||
yield tx
|
||||
if before_commit:
|
||||
raise RuntimeError("injected pre-commit interruption")
|
||||
if after_commit:
|
||||
raise RuntimeError("injected lost commit acknowledgement")
|
||||
|
||||
|
||||
def _store(db: Prisma, **faults: bool) -> BaselineAccountingStore:
|
||||
def transaction():
|
||||
return _transaction(db, **faults)
|
||||
|
||||
return BaselineAccountingStore(transaction)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def record() -> Callable[..., BaselineAccountingRecord]:
|
||||
run: Final = uuid.uuid4().hex
|
||||
marker: Final = CountedBreakpoint("prefix", 3600, 6000, ("prefix",), "content", ("content",))
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=6200, completion_tokens=30, total_tokens=6230,
|
||||
cache_creation_input_tokens=6000, cache_read_input_tokens=0,
|
||||
prompt_tokens_details={
|
||||
"text_tokens": 200, "cached_tokens": 0, "cache_creation_tokens": 6000,
|
||||
"cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6000},
|
||||
},
|
||||
)
|
||||
|
||||
def create(label: str = "first", started: float = 10000.0, identical: bool = True) -> BaselineAccountingRecord:
|
||||
return BaselineAccountingRecord(
|
||||
scope="autorouter-baseline:v3:" + run * 2, api_key=run, session_id=run,
|
||||
router_name="test-router", baseline_model="anthropic/claude-opus-5",
|
||||
observation=BaselineObservation(
|
||||
request_id=run + label, started_at=started, available_at=started + 0.1,
|
||||
outcome="complete", baseline_equivalent=identical, usage=usage,
|
||||
plan=CountedPromptCachePlan(6200, (marker,)), minimum_cache_tokens=4096,
|
||||
),
|
||||
pricing=BaselineCostSnapshot(
|
||||
model="claude-opus-5", provider="anthropic",
|
||||
prices=litellm.get_model_info("claude-opus-5", custom_llm_provider="anthropic"),
|
||||
actual_spend=0.17, actual_token_cost=0.17,
|
||||
),
|
||||
turn=AutoRouterTurnTransaction(
|
||||
api_key=run, session_id=run, router_name="test-router", router_type="heuristic",
|
||||
model="claude-opus-5", turn_at=datetime.fromtimestamp(started, timezone.utc),
|
||||
total_tokens=6230, spend=0.17, saved_spend=0.0, classifier_cost=0.0,
|
||||
covered=True, cache_hit=False, cache_ttl_seconds=3600, cache_touched=True,
|
||||
baseline_model="anthropic/claude-opus-5",
|
||||
),
|
||||
daily=DailyBaselineAttribution(
|
||||
date="2026-09-15", api_key=run, model="claude-opus-5", custom_llm_provider="anthropic",
|
||||
targets=tuple(DailyBaselineTarget(entity=entity, entity_id=run) for entity in ("user", "team", "org", "end_user", "agent", "tag")),
|
||||
),
|
||||
)
|
||||
|
||||
return create
|
||||
|
||||
|
||||
async def _log(db: Prisma, record: BaselineAccountingRecord) -> None:
|
||||
await db.execute_raw(
|
||||
'INSERT INTO "LiteLLM_SpendLogs" (request_id,call_type,api_key,spend,"startTime","endTime") '
|
||||
"VALUES ($1, 'anthropic_messages', $2, 0.17, to_timestamp($3::float8), to_timestamp($3::float8))",
|
||||
record.observation.request_id, record.api_key, record.observation.started_at,
|
||||
)
|
||||
|
||||
|
||||
async def _session(db: Prisma, record: BaselineAccountingRecord):
|
||||
rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterSession" WHERE api_key=$1', record.api_key)
|
||||
return rows[0]
|
||||
|
||||
|
||||
async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
store: Final = _store(db)
|
||||
late: Final = record("late", 10001.0)
|
||||
early: Final = record("early", identical=False)
|
||||
await _log(db, late)
|
||||
assert await store.append(late) == "recorded"
|
||||
assert await store.project(late.scope) == "published"
|
||||
before: Final = await _session(db, late)
|
||||
assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17
|
||||
assert before["saved_spend"] == 0.0
|
||||
await _log(db, early)
|
||||
assert await store.append(early) == "recorded"
|
||||
pending: Final = await _session(db, late)
|
||||
assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0
|
||||
assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0
|
||||
waiting: Final = await db.query_raw('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id)
|
||||
assert waiting[0]["metadata"]["autorouter_savings"] is None
|
||||
assert waiting[0]["metadata"]["autorouter_savings_estimate"]["reason"] == "pending_projection"
|
||||
assert await store.project(early.scope) == "published"
|
||||
after: Final = await _session(db, late)
|
||||
assert after["spend"] == 0.34 and after["turns"] == 2
|
||||
assert after["savings_estimated_actual_spend"] == 0.17 and after["savings_estimated_turns"] == 1
|
||||
logs: Final = await db.query_raw('SELECT spend, metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id)
|
||||
assert logs[0]["spend"] == 0.17
|
||||
assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled"
|
||||
assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"])
|
||||
for table in ("DailyUserSpend", "DailyTeamSpend", "DailyOrganizationSpend", "DailyEndUserSpend", "DailyAgentSpend", "DailyTagSpend"):
|
||||
rows: Final = await db.query_raw(f'SELECT spend,api_requests,autorouter_savings_spend FROM "LiteLLM_{table}" WHERE api_key=$1', late.api_key)
|
||||
assert rows[0]["spend"] == rows[0]["api_requests"] == 0
|
||||
assert rows[0]["autorouter_savings_spend"] == pytest.approx(after["saved_spend"])
|
||||
|
||||
|
||||
async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
event: Final = record()
|
||||
await _log(db, event)
|
||||
assert await _store(db, after_commit=True).append(event) == "unavailable"
|
||||
store: Final = _store(db)
|
||||
assert set(await asyncio.gather(*(store.append(event) for _ in range(4)))) == {"recorded"}
|
||||
assert await store.project(event.scope) == "published"
|
||||
assert await store.project(event.scope) == "unchanged"
|
||||
session: Final = await _session(db, event)
|
||||
assert session["turns"] == session["savings_estimated_turns"] == 1
|
||||
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.17
|
||||
|
||||
|
||||
async def test_publication_rollback_keeps_dirty_revision_for_retry(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
event: Final = record()
|
||||
await _log(db, event)
|
||||
store: Final = _store(db)
|
||||
assert await store.append(event) == "recorded"
|
||||
assert await _store(db, before_commit=True).project(event.scope) == "unavailable"
|
||||
session: Final = await _session(db, event)
|
||||
assert session["spend"] == 0.17 and session["savings_estimated_turns"] == 0
|
||||
revisions: Final = await db.query_raw('SELECT revision,published_revision FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope=$1', event.scope)
|
||||
assert revisions[0]["revision"] > revisions[0]["published_revision"]
|
||||
assert await store.project(event.scope) == "published"
|
||||
assert (await _session(db, event))["savings_estimated_turns"] == 1
|
||||
|
||||
|
||||
async def test_conflicting_duplicate_cannot_restore_an_observed_estimate(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
event: Final = record()
|
||||
await _log(db, event)
|
||||
store: Final = _store(db)
|
||||
assert await store.append(event) == "recorded"
|
||||
assert await store.project(event.scope) == "published"
|
||||
conflict: Final = event.model_copy(update={"observation": event.observation.model_copy(update={"baseline_equivalent": False, "started_at": 20000.0, "available_at": 20001.0})})
|
||||
assert await store.append(conflict) == "recorded"
|
||||
assert (await _session(db, event))["savings_estimated_turns"] == 0
|
||||
assert await store.append(event) == "recorded"
|
||||
assert await store.project(event.scope) == "published"
|
||||
session: Final = await _session(db, event)
|
||||
assert session["turns"] == 1 and session["savings_estimated_turns"] == 0
|
||||
rows: Final = await db.query_raw('SELECT publication FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', event.observation.request_id)
|
||||
assert json.loads(rows[0]["publication"])["reason"] == "conflicting_observation"
|
||||
|
||||
|
||||
async def test_retired_history_never_recreates_an_initial_zero(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
original: Final = record()
|
||||
await _log(db, original)
|
||||
store: Final = _store(db)
|
||||
assert await store.append(original) == "recorded"
|
||||
assert await store.project(original.scope) == "published"
|
||||
await db.execute_raw('UPDATE "LiteLLM_AutoRouterBaselineComparison" SET updated_at=to_timestamp(0) WHERE scope=$1', original.scope)
|
||||
await store.retire_before(datetime(2000, 1, 1, tzinfo=timezone.utc), 1000, 1000)
|
||||
next_turn: Final = record("after-retention", 20000.0)
|
||||
await _log(db, next_turn)
|
||||
assert await store.append(next_turn) == "retired"
|
||||
assert await store.project(original.scope) == "unchanged"
|
||||
after: Final = await _session(db, original)
|
||||
assert after["turns"] == 2 and after["spend"] == 0.34
|
||||
assert after["savings_estimated_turns"] == 1 and after["savings_estimated_actual_spend"] == 0.17
|
||||
|
||||
|
||||
async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_attribution(
|
||||
db: Prisma, record: Callable[..., BaselineAccountingRecord], monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import os
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
|
||||
event: Final = record("routed", identical=False)
|
||||
capture: Final = CapturedBaselineObservation(
|
||||
scope=event.scope, api_key=event.api_key, session_id=event.session_id,
|
||||
router_name=event.router_name, baseline_model=event.baseline_model,
|
||||
model=event.pricing.model, prices=event.pricing.prices, observation=event.observation,
|
||||
)
|
||||
metadata: Final = {
|
||||
"routing_decision": {"router_model_name": event.router_name, "savings_baseline_model": event.baseline_model},
|
||||
"usage_object": event.observation.usage.model_dump(),
|
||||
"cost_breakdown": {"input_cost": 0.16, "output_cost": 0.01},
|
||||
"autorouter_savings": None, "autorouter_savings_estimate": {"version": 3, "status": "unknown", "reason": "pending_projection"},
|
||||
"autorouter_baseline_observation": capture.model_dump_json(),
|
||||
}
|
||||
payload: Final = {
|
||||
"request_id": event.observation.request_id, "api_key": event.api_key, "session_id": event.session_id,
|
||||
"startTime": datetime.fromtimestamp(event.observation.started_at, timezone.utc).isoformat(),
|
||||
"endTime": datetime.fromtimestamp(event.observation.available_at, timezone.utc).isoformat(),
|
||||
"spend": 0.17, "prompt_tokens": 6200, "completion_tokens": 30, "model": event.pricing.model,
|
||||
"model_group": event.router_name, "model_id": "baseline", "custom_llm_provider": "anthropic",
|
||||
"call_type": "anthropic_messages", "status": "success", "metadata": json.dumps(metadata),
|
||||
"user": None, "team_id": "", "organization_id": "org", "agent_id": None,
|
||||
"end_user": "", "request_tags": '["tag","tag"]',
|
||||
}
|
||||
monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False)
|
||||
client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache()))
|
||||
writer: Final = DBSpendUpdateWriter()
|
||||
try:
|
||||
await client.db.connect()
|
||||
await _log(db, event)
|
||||
await writer._enqueue_autorouter_turn_transaction(payload, client)
|
||||
assert len(client.baseline_accounting_transactions) == 1
|
||||
queued: Final = client.baseline_accounting_transactions[0]
|
||||
assert queued.daily is not None
|
||||
assert [(target.entity, target.entity_id) for target in queued.daily.targets] == [
|
||||
("user", None), ("team", ""), ("org", "org"), ("tag", "tag"),
|
||||
]
|
||||
await writer.add_spend_log_transaction_to_daily_tag_transaction(payload, client)
|
||||
actual_tags: Final = await writer.daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
assert len(actual_tags) == 1
|
||||
assert next(iter(actual_tags.values()))["spend"] == 0.17
|
||||
durable: Final = BaselineAccountingStore.for_client(client)
|
||||
anchor: Final = record("anchor", 9999.0)
|
||||
await _log(db, anchor)
|
||||
assert await durable.append(anchor) == "recorded"
|
||||
assert await durable.append(queued) == "recorded"
|
||||
assert await durable.append(queued) == "recorded"
|
||||
assert await durable.project(queued.scope) == "published"
|
||||
session: Final = await _session(db, queued)
|
||||
assert session["turns"] == session["savings_estimated_turns"] == 2
|
||||
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34
|
||||
tag_rows: Final = await db.query_raw(
|
||||
'SELECT spend, api_requests, autorouter_savings_spend FROM "LiteLLM_DailyTagSpend" WHERE api_key=$1 AND tag=$2',
|
||||
queued.api_key, "tag",
|
||||
)
|
||||
assert session["saved_spend"] < 0
|
||||
assert len(tag_rows) == 1
|
||||
assert tag_rows[0]["autorouter_savings_spend"] == pytest.approx(session["saved_spend"])
|
||||
assert tag_rows[0]["spend"] == tag_rows[0]["api_requests"] == 0
|
||||
finally:
|
||||
await client.db.disconnect()
|
||||
103
tests/proxy_migration_tests/test_autorouter_baseline_state.py
Normal file
103
tests/proxy_migration_tests/test_autorouter_baseline_state.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Idempotent journal migration and primary transactional ownership."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Iterator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from psycopg import sql
|
||||
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.baseline_accounting import BaselineAccountingStore
|
||||
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
|
||||
_MIGRATION: Final = Path(__file__).parents[2] / (
|
||||
"litellm-proxy-extras/litellm_proxy_extras/migrations/20260915010000_add_autorouter_baseline_state/migration.sql"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database() -> Iterator[tuple[str, psycopg.Connection[tuple[object, ...]]]]:
|
||||
base: Final = os.environ["DATABASE_URL"].split("?")[0]
|
||||
schema: Final = f"baseline_{uuid4().hex}"
|
||||
with psycopg.connect(base, autocommit=True) as connection:
|
||||
connection.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
|
||||
connection.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(schema)))
|
||||
try:
|
||||
connection.execute(_MIGRATION.read_bytes())
|
||||
connection.execute(_MIGRATION.read_bytes())
|
||||
yield f"{base}?schema={schema}", connection
|
||||
finally:
|
||||
connection.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _client(env: pytest.MonkeyPatch, url: str, replica: str | None = None) -> AsyncGenerator[PrismaClient]:
|
||||
with env.context() as context:
|
||||
context.setenv("DATABASE_URL", url)
|
||||
context.delenv("DATABASE_URL_READ_REPLICA", raising=False)
|
||||
if replica is not None:
|
||||
context.setenv("DATABASE_URL_READ_REPLICA", replica)
|
||||
client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache()))
|
||||
try:
|
||||
await client.db.connect(timeout=timedelta(seconds=1))
|
||||
yield client
|
||||
finally:
|
||||
await client.db.disconnect()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_and_projector_use_the_primary_across_clients(
|
||||
database: tuple[str, psycopg.Connection[tuple[object, ...]]], monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
url, connection = database
|
||||
connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id TEXT PRIMARY KEY)')
|
||||
connection.execute('INSERT INTO "LiteLLM_AutoRouterBaselineComparison" '
|
||||
'(scope,api_key,session_id,router_name,initial_equivalent,revision) '
|
||||
"VALUES ('test','key','session','router',TRUE,1)")
|
||||
async with _client(monkeypatch, url, url.split("?")[0]) as first:
|
||||
assert await BaselineAccountingStore.for_client(first).project("test") == "published"
|
||||
async with _client(monkeypatch, url) as restarted:
|
||||
assert await BaselineAccountingStore.for_client(restarted).project("test") == "unchanged"
|
||||
assert connection.execute('SELECT revision=published_revision FROM "LiteLLM_AutoRouterBaselineComparison"').fetchone() == (True,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_outage_and_missing_table_are_unavailable(
|
||||
database: tuple[str, psycopg.Connection[tuple[object, ...]]], monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
url, connection = database
|
||||
async with _client(monkeypatch, "postgresql://unused:unused@127.0.0.1:1/unreachable", url) as degraded:
|
||||
assert isinstance(degraded.db, RoutingPrismaWrapper) and degraded.db.writer_unavailable
|
||||
assert await BaselineAccountingStore.for_client(degraded).project("scope") == "unavailable"
|
||||
connection.execute('DROP TABLE "LiteLLM_AutoRouterBaselineComparison"')
|
||||
async with _client(monkeypatch, url) as missing:
|
||||
assert await BaselineAccountingStore.for_client(missing).project("scope") == "unavailable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locked_projection_is_bounded_and_cancellation_propagates(
|
||||
database: tuple[str, psycopg.Connection[tuple[object, ...]]], monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
url, connection = database
|
||||
async with _client(monkeypatch, url) as client:
|
||||
store: Final = BaselineAccountingStore.for_client(client)
|
||||
with connection.transaction():
|
||||
connection.execute('LOCK TABLE "LiteLLM_AutoRouterBaselineComparison" IN ACCESS EXCLUSIVE MODE')
|
||||
started: Final = time.monotonic()
|
||||
assert await store.project("scope") == "unavailable"
|
||||
assert time.monotonic() - started < 2
|
||||
pending: Final = asyncio.create_task(store.project("scope"))
|
||||
await asyncio.sleep(0.01)
|
||||
pending.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await pending
|
||||
assert await store.project("scope") == "unchanged"
|
||||
|
|
@ -37,10 +37,15 @@ class MockPrismaClient:
|
|||
self.daily_user_spend_transactions = {}
|
||||
self.tool_usage_transactions = []
|
||||
self.autorouter_turn_transactions = []
|
||||
self.baseline_accounting_transactions = []
|
||||
self.baseline_accounting_lock = asyncio.Lock()
|
||||
self.spend_log_flush_requested = None
|
||||
self.db.tx = MagicMock()
|
||||
self.db.tx.return_value.__aenter__ = AsyncMock(return_value=self.db)
|
||||
self.db.tx.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||
self.db.query_raw.return_value = []
|
||||
|
||||
# Add locks for the transaction queues (matches real PrismaClient)
|
||||
import asyncio
|
||||
|
||||
self._spend_log_transactions_lock = asyncio.Lock()
|
||||
self._tool_usage_transactions_lock = asyncio.Lock()
|
||||
self._autorouter_turn_transactions_lock = asyncio.Lock()
|
||||
|
|
|
|||
|
|
@ -1067,6 +1067,7 @@ async def test_afile_content_passes_trusted_model_credentials_to_router():
|
|||
managed_files = _make_managed_files_instance()
|
||||
unified_file_id = "unified-file-id"
|
||||
s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out"
|
||||
managed_files.get_unified_file_id = AsyncMock(return_value=None)
|
||||
managed_files.get_model_file_id_mapping = AsyncMock(
|
||||
return_value={unified_file_id: {"model-123": s3_uri}}
|
||||
)
|
||||
|
|
@ -1238,6 +1239,7 @@ async def test_afile_content_bedrock_unified_id_end_to_end(monkeypatch):
|
|||
managed_files = _make_managed_files_instance()
|
||||
unified_file_id = "unified-file-id"
|
||||
s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out"
|
||||
managed_files.get_unified_file_id = AsyncMock(return_value=None)
|
||||
managed_files.get_model_file_id_mapping = AsyncMock(
|
||||
return_value={unified_file_id: {"model-123": s3_uri}}
|
||||
)
|
||||
|
|
@ -1268,6 +1270,7 @@ async def test_afile_content_error_reports_unified_id_not_provider_uri():
|
|||
managed_files = _make_managed_files_instance()
|
||||
unified_file_id = "litellm_proxy_unified_id_abc"
|
||||
s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out"
|
||||
managed_files.get_unified_file_id = AsyncMock(return_value=None)
|
||||
managed_files.get_model_file_id_mapping = AsyncMock(
|
||||
return_value={unified_file_id: {"model-123": s3_uri}}
|
||||
)
|
||||
|
|
@ -1732,6 +1735,40 @@ async def test_batch_retrieve_hook_does_not_claim_attribution():
|
|||
assert managed_files.store_unified_object_id.await_args.kwargs["persist_attribution"] is False
|
||||
|
||||
|
||||
def _unified_batch_id(llm_batch_id: str) -> str:
|
||||
decoded = f"litellm_proxy;model_id:my-vllm;llm_batch_id:{llm_batch_id}"
|
||||
return base64.urlsafe_b64encode(decoded.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"llm_batch_id, stores",
|
||||
[("litellm_batch_abc", False), ("batch_abc", True)],
|
||||
ids=["litellm-executed batch is left alone", "provider batch is still stored"],
|
||||
)
|
||||
async def test_post_call_hook_leaves_litellm_executed_batches_untouched(llm_batch_id: str, stores: bool):
|
||||
managed_files = _make_managed_files_instance()
|
||||
response = _make_batch_response(status="in_progress", output_file_id=None)
|
||||
response.id = _unified_batch_id(llm_batch_id)
|
||||
response._hidden_params = {
|
||||
"unified_batch_id": response.id,
|
||||
"model_id": "my-vllm",
|
||||
"model_name": "hosted_vllm/qwen",
|
||||
}
|
||||
original_id = response.id
|
||||
|
||||
returned = await managed_files.async_post_call_success_hook(
|
||||
data={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-the-poller", user_id="bob", parent_otel_span=None),
|
||||
response=response,
|
||||
)
|
||||
|
||||
assert returned is response
|
||||
assert managed_files.store_unified_object_id.await_count == (1 if stores else 0)
|
||||
if not stores:
|
||||
assert response.id == original_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_afile_delete_passes_trusted_model_credentials_to_router():
|
||||
"""
|
||||
|
|
@ -1743,6 +1780,7 @@ async def test_afile_delete_passes_trusted_model_credentials_to_router():
|
|||
managed_files = _make_managed_files_instance()
|
||||
unified_file_id = "unified-file-id"
|
||||
s3_uri = "s3://my-bucket/litellm-bedrock-files/job-123/input.jsonl"
|
||||
managed_files.get_unified_file_id = AsyncMock(return_value=None)
|
||||
managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}})
|
||||
managed_files.delete_unified_file_id = AsyncMock(return_value=_make_file_object(unified_file_id))
|
||||
|
||||
|
|
@ -1809,6 +1847,7 @@ async def test_afile_delete_bedrock_unified_id_end_to_end(monkeypatch):
|
|||
managed_files = _make_managed_files_instance()
|
||||
unified_file_id = "unified-file-id"
|
||||
s3_uri = "s3://my-bucket/litellm-bedrock-files/job-123/input.jsonl"
|
||||
managed_files.get_unified_file_id = AsyncMock(return_value=None)
|
||||
managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}})
|
||||
managed_files.delete_unified_file_id = AsyncMock(return_value=_make_file_object(unified_file_id))
|
||||
|
||||
|
|
@ -1827,3 +1866,147 @@ async def test_afile_delete_bedrock_unified_id_end_to_end(monkeypatch):
|
|||
assert response.id == unified_file_id
|
||||
assert response.model_dump() == {"id": unified_file_id, "object": "file", "deleted": True}
|
||||
managed_files.delete_unified_file_id.assert_awaited_once_with(unified_file_id, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_afile_delete_storage_backed_row_deletes_stored_content_not_provider_files():
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
from openai.types import FileDeleted
|
||||
|
||||
from litellm.caching import DualCache
|
||||
from litellm.models.managed_files import LiteLLM_ManagedFileTable
|
||||
|
||||
storage_url = "litellm_db://content-row-1"
|
||||
unified_file_id = _managed_deletion_file_id(storage_url)
|
||||
row = LiteLLM_ManagedFileTable(
|
||||
unified_file_id=unified_file_id,
|
||||
model_mappings={"vllm-batch": storage_url},
|
||||
flat_model_file_ids=[storage_url],
|
||||
file_object=_make_file_object(unified_file_id),
|
||||
storage_backend="litellm_db",
|
||||
storage_url=storage_url,
|
||||
)
|
||||
file_table = MagicMock(find_first=AsyncMock(return_value=row), delete=AsyncMock())
|
||||
content_table = MagicMock(delete=AsyncMock())
|
||||
managed_files = _PROXY_LiteLLMManagedFiles(
|
||||
internal_usage_cache=DualCache(),
|
||||
prisma_client=MagicMock(
|
||||
db=MagicMock(litellm_managedfiletable=file_table, litellm_managedfilecontenttable=content_table)
|
||||
),
|
||||
)
|
||||
router = MagicMock(
|
||||
get_deployment_credentials_with_provider=MagicMock(return_value=None),
|
||||
afile_delete=AsyncMock(),
|
||||
)
|
||||
|
||||
response = await managed_files.afile_delete(
|
||||
file_id=unified_file_id,
|
||||
litellm_parent_otel_span=None,
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
content_table.delete.assert_awaited_once_with(where={"id": "content-row-1"})
|
||||
router.afile_delete.assert_not_awaited()
|
||||
file_table.delete.assert_awaited_once_with(where={"unified_file_id": unified_file_id})
|
||||
assert response == FileDeleted(id=unified_file_id, object="file", deleted=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_afile_content_storage_backed_row_returns_stored_bytes_not_provider_content():
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
from prisma import Base64
|
||||
|
||||
from litellm.caching import DualCache
|
||||
from litellm.models.managed_files import LiteLLM_ManagedFileTable
|
||||
|
||||
storage_url = "litellm_db://content-row-1"
|
||||
unified_file_id = _managed_deletion_file_id(storage_url)
|
||||
stored_bytes = b'{"custom_id": "line-1", "method": "POST", "url": "/v1/chat/completions", "body": {}}\n'
|
||||
row = LiteLLM_ManagedFileTable(
|
||||
unified_file_id=unified_file_id,
|
||||
model_mappings={"vllm-batch": storage_url},
|
||||
flat_model_file_ids=[storage_url],
|
||||
file_object=_make_file_object(unified_file_id),
|
||||
storage_backend="litellm_db",
|
||||
storage_url=storage_url,
|
||||
)
|
||||
file_table = MagicMock(find_first=AsyncMock(return_value=row))
|
||||
content_table = MagicMock(find_unique=AsyncMock(return_value=MagicMock(content=Base64.encode(stored_bytes))))
|
||||
managed_files = _PROXY_LiteLLMManagedFiles(
|
||||
internal_usage_cache=DualCache(),
|
||||
prisma_client=MagicMock(
|
||||
db=MagicMock(litellm_managedfiletable=file_table, litellm_managedfilecontenttable=content_table)
|
||||
),
|
||||
)
|
||||
router = MagicMock(
|
||||
get_deployment_credentials_with_provider=MagicMock(return_value=None),
|
||||
afile_content=AsyncMock(),
|
||||
)
|
||||
|
||||
response = await managed_files.afile_content(
|
||||
file_id=unified_file_id,
|
||||
litellm_parent_otel_span=None,
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
assert response.content == stored_bytes
|
||||
content_table.find_unique.assert_awaited_once_with(where={"id": "content-row-1"})
|
||||
router.afile_content.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_unified_object_id_batch_processed_is_written_only_when_asked():
|
||||
managed_files, mock_prisma = _make_object_store_instance()
|
||||
upsert = mock_prisma.db.litellm_managedobjecttable.upsert
|
||||
creator = UserAPIKeyAuth(api_key="sk-creator", user_id="alice", team_id="team-alpha", parent_otel_span=None)
|
||||
|
||||
await managed_files.store_unified_object_id(
|
||||
unified_object_id="uoi-processed",
|
||||
file_object=_make_batch_response(status="completed"),
|
||||
litellm_parent_otel_span=None,
|
||||
model_object_id="batch-processed",
|
||||
file_purpose="batch",
|
||||
user_api_key_dict=creator,
|
||||
batch_processed=True,
|
||||
)
|
||||
await managed_files.store_unified_object_id(
|
||||
unified_object_id="uoi-default",
|
||||
file_object=_make_batch_response(status="completed"),
|
||||
litellm_parent_otel_span=None,
|
||||
model_object_id="batch-default",
|
||||
file_purpose="batch",
|
||||
user_api_key_dict=creator,
|
||||
)
|
||||
|
||||
processed_create, default_create = (call.kwargs["data"]["create"] for call in upsert.await_args_list)
|
||||
assert processed_create["batch_processed"] is True
|
||||
assert default_create["batch_processed"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_unified_file_id_caches_the_storage_location_the_db_row_gets():
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
|
||||
from litellm.caching import DualCache
|
||||
|
||||
file_table = MagicMock(upsert=AsyncMock(), find_first=AsyncMock(side_effect=AssertionError("cache miss")))
|
||||
managed_files = _PROXY_LiteLLMManagedFiles(
|
||||
internal_usage_cache=DualCache(),
|
||||
prisma_client=MagicMock(db=MagicMock(litellm_managedfiletable=file_table)),
|
||||
)
|
||||
stored = _make_file_object("file-kept").model_copy(update={"purpose": "batch"})
|
||||
stored._hidden_params = {"storage_backend": "litellm_db", "storage_url": "litellm_db://content-row-1"}
|
||||
|
||||
await managed_files.store_unified_file_id(
|
||||
file_id="unified-kept",
|
||||
file_object=stored,
|
||||
litellm_parent_otel_span=None,
|
||||
model_mappings={"vllm-batch": "litellm_db://content-row-1"},
|
||||
user_api_key_dict=_make_user_api_key_dict(),
|
||||
)
|
||||
cached = await managed_files.get_unified_file_id("unified-kept")
|
||||
|
||||
assert cached is not None
|
||||
assert (cached.storage_backend, cached.storage_url) == ("litellm_db", "litellm_db://content-row-1")
|
||||
create_data = file_table.upsert.await_args.kwargs["data"]["create"]
|
||||
assert (create_data["storage_backend"], create_data["storage_url"]) == ("litellm_db", "litellm_db://content-row-1")
|
||||
|
|
|
|||
|
|
@ -3532,6 +3532,24 @@ def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
|
|||
assert merged.get("applied_guardrails") == ["pam-ethical-request"]
|
||||
|
||||
|
||||
def test_get_standard_logging_metadata_merges_recorded_applied_guardrails():
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
|
||||
result = StandardLoggingPayloadSetup.get_standard_logging_metadata(
|
||||
metadata={"applied_guardrails": ["blocker"]},
|
||||
litellm_params={},
|
||||
applied_guardrails=["guard-a", "blocker", "guard-b"],
|
||||
)
|
||||
assert result["applied_guardrails"] == ["guard-a", "blocker", "guard-b"]
|
||||
|
||||
result = StandardLoggingPayloadSetup.get_standard_logging_metadata(
|
||||
metadata={"applied_guardrails": ["blocker"]},
|
||||
litellm_params={},
|
||||
applied_guardrails=["guard-a"],
|
||||
)
|
||||
assert result["applied_guardrails"] == ["guard-a", "blocker"]
|
||||
|
||||
|
||||
def test_function_setup_metadata_takes_precedence_over_litellm_metadata():
|
||||
"""
|
||||
Test that when BOTH metadata and litellm_metadata are present (e.g., user sets
|
||||
|
|
|
|||
|
|
@ -124,22 +124,32 @@ def test_calculate_usage_prefers_served_speed_from_response_usage():
|
|||
assert no_response_speed.speed == "fast"
|
||||
|
||||
|
||||
def test_streaming_iterator_persists_served_speed_across_usage_chunks():
|
||||
@pytest.mark.parametrize("input_update, expected_fresh", [({}, 1000), ({"input_tokens": 0}, 0), ({"input_tokens": 2000}, 2000)])
|
||||
def test_streaming_iterator_persists_cumulative_usage_across_partial_chunks(input_update, expected_fresh):
|
||||
"""
|
||||
Only ``message_start`` usage carries the served speed; the final
|
||||
``message_delta`` usage does not. The iterator must remember the served
|
||||
value so the last usage chunk, which wins in the stream chunk builder, does
|
||||
not fall back to the requested speed.
|
||||
Omitted input/cache/pricing fields retain their last cumulative values;
|
||||
explicit input updates, including zero, replace them.
|
||||
"""
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
iterator = ModelResponseIterator(None, sync_stream=True, speed="fast")
|
||||
|
||||
start_usage = iterator._handle_usage({"input_tokens": 12, "output_tokens": 1, "speed": "standard"})
|
||||
delta_usage = iterator._handle_usage({"output_tokens": 5})
|
||||
start_usage = iterator._handle_usage({
|
||||
"input_tokens": 1000, "output_tokens": 1, "speed": "standard", "inference_geo": "us",
|
||||
"cache_creation_input_tokens": 3000, "cache_read_input_tokens": 2000,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3000},
|
||||
})
|
||||
delta_usage = iterator._handle_usage({"output_tokens": 5, **input_update})
|
||||
|
||||
assert start_usage.speed == "standard"
|
||||
assert delta_usage.speed == "standard"
|
||||
assert delta_usage.inference_geo == "us"
|
||||
assert delta_usage.prompt_tokens == expected_fresh + 5000
|
||||
assert delta_usage.completion_tokens == 5
|
||||
details = delta_usage.prompt_tokens_details
|
||||
assert (details.text_tokens, details.cached_tokens, details.cache_creation_tokens) == (expected_fresh, 2000, 3000)
|
||||
assert details.cache_creation_token_details.ephemeral_1h_input_tokens == 3000
|
||||
assert start_usage.prompt_tokens_details.text_tokens == 1000
|
||||
|
||||
|
||||
def test_calculate_usage_aggregates_cache_creation_split_across_iterations():
|
||||
|
|
|
|||
|
|
@ -15,11 +15,17 @@ from litellm.caching.llm_caching_handler import LLMClientCache
|
|||
from litellm.llms.anthropic.count_tokens import handler as count_handler
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION
|
||||
from litellm.llms.anthropic.prompt_cache_prediction import (
|
||||
CountedPromptCachePlan,
|
||||
NativePredictionTarget,
|
||||
PromptCachePlan,
|
||||
UnsupportedCachePlan,
|
||||
cache_scope,
|
||||
count_cache_plan,
|
||||
count_prompt_tokens,
|
||||
parse_cache_plan,
|
||||
parse_observed_cache,
|
||||
parse_prompt,
|
||||
resolve_baseline_prediction_target,
|
||||
resolve_prediction_target,
|
||||
supported_prediction_headers,
|
||||
)
|
||||
|
|
@ -207,3 +213,232 @@ async def test_named_credential_is_explicitly_unsupported_before_count(
|
|||
assert arm.cache_state == "unknown"
|
||||
assert arm.reason == "unsupported_deployment_configuration"
|
||||
assert arm.estimate is None and arm.cold is None and arm.warm is None
|
||||
|
||||
|
||||
def _cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan:
|
||||
plan: Final = parse_cache_plan(body)
|
||||
assert isinstance(plan, PromptCachePlan)
|
||||
return plan
|
||||
|
||||
|
||||
def _text(text: str, ttl: str | None = None) -> dict[str, JsonValue]:
|
||||
return {"type": "text", "text": text,
|
||||
**({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})}
|
||||
|
||||
|
||||
def _prompt(*blocks: dict[str, JsonValue], role: str = "user", **options: JsonValue) -> dict[str, JsonValue]:
|
||||
return {**options, "messages": [{"role": role, "content": list(blocks)}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, supported", [("", False), (" \t", False), ("Context", True)])
|
||||
def test_public_predictor_preserves_string_message_policy(text: str, supported: bool) -> None:
|
||||
body: Final = _body()
|
||||
messages: Final = body["messages"]
|
||||
assert isinstance(messages, list)
|
||||
request: Final[dict[str, JsonValue]] = {**body, "messages": [{"role": "user", "content": text}, *messages]}
|
||||
assert (parse_prompt(request) is not None) is supported
|
||||
|
||||
|
||||
def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None:
|
||||
body: Final = _prompt(
|
||||
_text("First turn", "5m"), system=[_text("Stable instructions", "1h")],
|
||||
tools=[{"name": "lookup", "input_schema": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"}}],
|
||||
)
|
||||
plan: Final = _cache_plan(body)
|
||||
changed: Final = _cache_plan({**body, "system": [_text("Changed instructions", "1h")]})
|
||||
assert tuple(marker.ttl_seconds for marker in plan.breakpoints) == (3600, 3600, 300)
|
||||
assert plan.breakpoints[0].fingerprint == changed.breakpoints[0].fingerprint
|
||||
assert all(left.fingerprint != right.fingerprint for left, right
|
||||
in zip(plan.breakpoints[1:], changed.breakpoints[1:]))
|
||||
assert plan.breakpoints[0].prefix_body == {
|
||||
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
|
||||
"messages": [],
|
||||
}
|
||||
assert parse_prompt(body) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, added, matches", [
|
||||
("text", 19, True), ("text", 20, False), ("tool_use", 30, True),
|
||||
("tool_result", 30, True),
|
||||
])
|
||||
def test_cache_plan_lookback_counts_native_positions(
|
||||
kind: str, added: int, matches: bool,
|
||||
) -> None:
|
||||
previous: Final = _cache_plan(_body())
|
||||
appended: Final[list[dict[str, JsonValue]]] = [
|
||||
{"type": "tool_use", "id": f"tool_{index}", "name": "lookup", "input": {}}
|
||||
if kind == "tool_use" else
|
||||
{"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"}
|
||||
if kind == "tool_result" else
|
||||
{"type": "text", "text": f"Added {index}"}
|
||||
for index in range(added)
|
||||
]
|
||||
current: Final = _cache_plan({**_body(), **_prompt(
|
||||
_text("A cacheable prefix"), *appended[:-1],
|
||||
{**appended[-1], "cache_control": {"type": "ephemeral"}},
|
||||
)})
|
||||
assert (previous.breakpoints[0].fingerprint
|
||||
in current.breakpoints[0].lookback_fingerprints) is matches
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change, same_prefix, same_content", [
|
||||
("tool_order", False, False), ("effort", False, False),
|
||||
("standard_speed", True, True), ("ttl", False, True),
|
||||
])
|
||||
def test_cache_plan_identity_respects_settings_and_preserves_content(
|
||||
change: str, same_prefix: bool, same_content: bool,
|
||||
) -> None:
|
||||
tool_input: Final[dict[str, JsonValue]] = {"a": 1, "b": 2, "cache_control": {"ttl": "user-data"}}
|
||||
block: Final[dict[str, JsonValue]] = {
|
||||
"type": "tool_use", "id": "tool_1", "name": "lookup", "input": tool_input,
|
||||
"cache_control": {"type": "ephemeral", "ttl": "5m"},
|
||||
}
|
||||
changed_block: Final = (
|
||||
{**block, "input": dict(reversed(tool_input.items()))} if change == "tool_order" else
|
||||
{**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} if change == "ttl" else block
|
||||
)
|
||||
before: Final = _cache_plan(_prompt(block, role="assistant", output_config={"effort": "low"})).breakpoints[0]
|
||||
after: Final = _cache_plan(_prompt(
|
||||
changed_block, role="assistant", output_config={"effort": "high" if change == "effort" else "low"},
|
||||
**({"speed": "standard"} if change == "standard_speed" else {}),
|
||||
)).breakpoints[0]
|
||||
assert (before.fingerprint == after.fingerprint) is same_prefix
|
||||
assert (before.fingerprint in after.lookback_fingerprints) is same_prefix
|
||||
assert (before.content_fingerprint == after.content_fingerprint) is same_content
|
||||
assert (before.content_fingerprint in after.lookback_content_fingerprints) is same_content
|
||||
assert "user-data" in json.dumps(dict(before.prefix_body))
|
||||
assert not supported_prediction_headers({"anthropic-beta": "fast-mode-2026-02-01"})
|
||||
|
||||
|
||||
def test_cache_plan_automatic_cache_and_thinking_use_last_cacheable_block() -> None:
|
||||
body: Final = _prompt(
|
||||
_text("A stable answer"), {"type": "thinking", "thinking": "Thinking", "signature": "signature"},
|
||||
role="assistant", thinking={"type": "adaptive"}, cache_control={"type": "ephemeral", "ttl": "1h"},
|
||||
)
|
||||
plan: Final = _cache_plan(body)
|
||||
assert len(plan.breakpoints) == 1
|
||||
assert plan.breakpoints[0].ttl_seconds == 3600
|
||||
assert plan.breakpoints[0].prefix_body == {
|
||||
"thinking": {"type": "adaptive"},
|
||||
"messages": [{"role": "assistant", "content": [
|
||||
{"type": "text", "text": "A stable answer"},
|
||||
]}],
|
||||
}
|
||||
assert parse_prompt(body) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body, reason", [
|
||||
(_prompt({"type": "image"}), "unsupported_prompt_shape"),
|
||||
({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"),
|
||||
({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}},
|
||||
"conflicting_cache_ttl"),
|
||||
(_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"),
|
||||
])
|
||||
def test_cache_plan_unsupported_is_explicit(
|
||||
body: Mapping[str, JsonValue], reason: str,
|
||||
) -> None:
|
||||
result: Final = parse_cache_plan(body)
|
||||
assert isinstance(result, UnsupportedCachePlan)
|
||||
assert result.reason == reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model, counts, reason", [
|
||||
(None, (100, 150, 200), None),
|
||||
(None, (100, 201, 200), "inconsistent_prefix_token_count"),
|
||||
(None, (151, 150, 200), "inconsistent_prefix_token_count"),
|
||||
(None, (None, 150, 200), "token_count_unavailable"),
|
||||
("claude-opus-5", (100, 150, 200), None),
|
||||
("claude-sonnet-5", (100, 150, 200), None),
|
||||
("declared-cache-model", (100, 150, 200), None),
|
||||
("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"),
|
||||
("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
|
||||
("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
|
||||
])
|
||||
async def test_cache_plan_count_conserves_total_and_rejects_unknown(
|
||||
model: str | None, counts: tuple[int | None, int | None, int], reason: str | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "declared-cache-model", {
|
||||
"litellm_provider": "anthropic", "mode": "chat", "supports_thinking_cache_preservation": True,
|
||||
})
|
||||
plan: Final = _cache_plan(_prompt(
|
||||
{"type": "thinking", "thinking": "Retained thought", "signature": "signature"}
|
||||
if model else _text("first", "5m"),
|
||||
_text("second", "5m"), _text("uncached"), role="assistant" if model else "user",
|
||||
))
|
||||
|
||||
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
|
||||
assert reason != "unsupported_thinking_cache_semantics", "Unverified thinking retention must skip counting"
|
||||
if body is plan.full_body:
|
||||
return counts[2]
|
||||
return counts[0] if body is plan.breakpoints[0].prefix_body else counts[1]
|
||||
|
||||
result: Final = await count_cache_plan(model or _MODEL, _KEY, plan, count)
|
||||
if reason is not None:
|
||||
assert isinstance(result, UnsupportedCachePlan)
|
||||
assert result.reason == reason
|
||||
else:
|
||||
assert isinstance(result, CountedPromptCachePlan)
|
||||
assert result.total_tokens == 200
|
||||
assert tuple(marker.prefix_tokens for marker in result.breakpoints) == ((100,) if model else (100, 150))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("section", ["system", "tools"])
|
||||
@pytest.mark.parametrize("rejects_prefix", (False, True))
|
||||
async def test_native_count_preserves_settings_and_requires_every_prefix(
|
||||
section: str, rejects_prefix: bool, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
params: Final = LiteLLM_Params(
|
||||
model=f"anthropic/{_MODEL}", api_key=_KEY,
|
||||
api_base="https://gateway.example/v1/messages",
|
||||
)
|
||||
target: Final = resolve_baseline_prediction_target(params)
|
||||
assert isinstance(target, NativePredictionTarget)
|
||||
assert target.api_base == params.api_base
|
||||
assert not isinstance(resolve_prediction_target(params), NativePredictionTarget)
|
||||
body: Final = _body()
|
||||
marker: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"}
|
||||
body[section] = ([_text("A cached system", "1h")] if section == "system" else [{
|
||||
"name": "lookup", "input_schema": {"type": "object"}, "cache_control": marker,
|
||||
}])
|
||||
plan: Final = _cache_plan({**body, **_prompt(
|
||||
_text("A later prefix", "5m"), _text("An uncached suffix"),
|
||||
thinking={"type": "adaptive"}, tool_choice={"type": "auto"}, output_config={"effort": "high"},
|
||||
)})
|
||||
assert len(plan.breakpoints) == 2
|
||||
assert plan.breakpoints[0].prefix_body["messages"] == []
|
||||
|
||||
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
|
||||
return await count_prompt_tokens(
|
||||
model, api_key, {**body, "max_tokens": 100}, api_base=target.api_base,
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
endpoint: Final = "https://gateway.example/v1/messages/count_tokens"
|
||||
routes: Final = tuple(
|
||||
upstream.post(endpoint, json={**body, "model": _MODEL}).respond(
|
||||
400 if rejects_prefix and index == 1 else 200,
|
||||
json={"detail": {"error": "messages parameter is required"}}
|
||||
if rejects_prefix and index == 1 else {"input_tokens": tokens},
|
||||
)
|
||||
for index, (body, tokens) in enumerate((
|
||||
(plan.full_body, 6000), (plan.breakpoints[0].prefix_body, 5000),
|
||||
(plan.breakpoints[1].prefix_body, 5800),
|
||||
))
|
||||
)
|
||||
unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1})
|
||||
result: Final = await count_cache_plan(target.model, target.api_key, plan, count)
|
||||
|
||||
if rejects_prefix:
|
||||
assert result == UnsupportedCachePlan("token_count_unavailable")
|
||||
else:
|
||||
assert isinstance(result, CountedPromptCachePlan)
|
||||
assert result.total_tokens == 6000
|
||||
assert tuple(marker.prefix_tokens for marker in result.breakpoints) == (5000, 5800)
|
||||
assert tuple(route.call_count for route in routes) == (1, 1, 1)
|
||||
assert unexpected.call_count == 0
|
||||
|
|
|
|||
|
|
@ -335,6 +335,129 @@ class TestAzureToolSchemaCombinatorFlattening:
|
|||
assert request["temperature"] == 0.2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", ["none", "auto"])
|
||||
def test_azure_drops_tool_choice_without_tools_or_functions(tool_choice: str) -> None:
|
||||
optional_params = {"tool_choice": tool_choice, "temperature": 0.2}
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "tool_choice" not in request
|
||||
assert request["temperature"] == 0.2
|
||||
assert optional_params["tool_choice"] == tool_choice
|
||||
|
||||
|
||||
def test_azure_tools_empty_drops_tool_choice() -> None:
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"tools": [], "tool_choice": "auto"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["tools"] == []
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
def test_azure_functions_empty_drops_tool_choice() -> None:
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"functions": [], "tool_choice": "none"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["functions"] == []
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
def test_azure_preserves_tool_choice_with_tools() -> None:
|
||||
tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"tools": tools, "tool_choice": "auto"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["tools"] == tools
|
||||
assert request["tool_choice"] == "auto"
|
||||
|
||||
|
||||
def test_azure_preserves_tool_choice_with_legacy_functions() -> None:
|
||||
functions = [{"name": "get_weather", "parameters": {}}]
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"functions": functions, "tool_choice": "auto"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["functions"] == functions
|
||||
assert request["tool_choice"] == "auto"
|
||||
|
||||
|
||||
def test_azure_preserves_function_call_without_tools() -> None:
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"function_call": "none", "tool_choice": "auto"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["function_call"] == "none"
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
def test_azure_gpt5_drops_tool_choice_without_tools() -> None:
|
||||
request = AzureOpenAIGPT5Config().transform_request(
|
||||
model="gpt5_series/gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"tool_choice": "none"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["model"] == "gpt-5.6-sol"
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_async_transform_drops_tool_choice_without_tools() -> None:
|
||||
request = await AzureOpenAIConfig().async_transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"tool_choice": "none"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_gpt5_async_transform_drops_tool_choice_without_tools() -> None:
|
||||
request = await AzureOpenAIGPT5Config().async_transform_request(
|
||||
model="gpt5_series/gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"tool_choice": "auto"},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["model"] == "gpt-5.6-sol"
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
def test_transform_request_strips_litellm_format_from_managed_file_id():
|
||||
import base64
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from prisma import Base64
|
||||
from prisma.errors import RecordNotFoundError
|
||||
|
||||
from litellm.llms.base_llm.files.litellm_db_storage_backend import (
|
||||
LITELLM_DB_STORAGE_URL_PREFIX,
|
||||
LiteLLMDbStorageBackend,
|
||||
storage_url_to_row_id,
|
||||
)
|
||||
|
||||
|
||||
def _backend_with_table():
|
||||
table = MagicMock(create=AsyncMock(), find_unique=AsyncMock(), delete=AsyncMock())
|
||||
prisma_client = MagicMock(db=MagicMock(litellm_managedfilecontenttable=table))
|
||||
return LiteLLMDbStorageBackend(prisma_client), table
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_stores_bytes_and_returns_prefixed_row_id():
|
||||
backend, table = _backend_with_table()
|
||||
table.create.return_value = SimpleNamespace(id="row-1")
|
||||
content = b"\x00\x01binary jsonl\n"
|
||||
|
||||
storage_url = await backend.upload_file(file_content=content, filename="input.jsonl", content_type="text/plain")
|
||||
|
||||
assert storage_url == f"{LITELLM_DB_STORAGE_URL_PREFIX}row-1"
|
||||
stored = table.create.await_args.kwargs["data"]["content"]
|
||||
assert isinstance(stored, Base64)
|
||||
assert stored.decode() == content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_returns_exact_bytes_of_the_row():
|
||||
backend, table = _backend_with_table()
|
||||
content = b'{"custom_id": "1"}\n'
|
||||
table.find_unique.return_value = SimpleNamespace(id="row-1", content=Base64.encode(content))
|
||||
|
||||
downloaded = await backend.download_file(f"{LITELLM_DB_STORAGE_URL_PREFIX}row-1")
|
||||
|
||||
assert downloaded == content
|
||||
table.find_unique.assert_awaited_once_with(where={"id": "row-1"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_missing_row_raises_value_error_naming_the_url():
|
||||
backend, table = _backend_with_table()
|
||||
table.find_unique.return_value = None
|
||||
storage_url = f"{LITELLM_DB_STORAGE_URL_PREFIX}missing"
|
||||
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
await backend.download_file(storage_url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_rejects_url_without_prefix_before_touching_the_db():
|
||||
backend, table = _backend_with_table()
|
||||
|
||||
with pytest.raises(ValueError, match="https://elsewhere/blob"):
|
||||
await backend.download_file("https://elsewhere/blob")
|
||||
|
||||
table.find_unique.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_removes_the_parsed_row():
|
||||
backend, table = _backend_with_table()
|
||||
|
||||
await backend.delete_file(f"{LITELLM_DB_STORAGE_URL_PREFIX}row-1")
|
||||
|
||||
table.delete.assert_awaited_once_with(where={"id": "row-1"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_tolerates_a_row_that_is_already_gone():
|
||||
backend, table = _backend_with_table()
|
||||
table.delete.side_effect = RecordNotFoundError({"user_facing_error": {"message": "gone"}})
|
||||
|
||||
await backend.delete_file(f"{LITELLM_DB_STORAGE_URL_PREFIX}row-1")
|
||||
|
||||
table.delete.assert_awaited_once_with(where={"id": "row-1"})
|
||||
|
||||
|
||||
def test_storage_url_to_row_id_round_trips():
|
||||
assert storage_url_to_row_id(f"{LITELLM_DB_STORAGE_URL_PREFIX}abc-123") == "abc-123"
|
||||
|
||||
|
||||
def test_storage_url_to_row_id_rejects_foreign_urls():
|
||||
with pytest.raises(ValueError, match="s3://bucket/key"):
|
||||
storage_url_to_row_id("s3://bucket/key")
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.base_llm.files.litellm_db_storage_backend import (
|
||||
LITELLM_DB_STORAGE_BACKEND_NAME,
|
||||
LITELLM_DB_STORAGE_URL_PREFIX,
|
||||
LiteLLMDbStorageBackend,
|
||||
)
|
||||
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_db_backend_stores_through_the_given_prisma_client():
|
||||
table = MagicMock(create=AsyncMock(return_value=SimpleNamespace(id="row-1")))
|
||||
prisma_client = MagicMock(db=MagicMock(litellm_managedfilecontenttable=table))
|
||||
|
||||
backend = get_storage_backend(LITELLM_DB_STORAGE_BACKEND_NAME, prisma_client=prisma_client)
|
||||
|
||||
assert isinstance(backend, LiteLLMDbStorageBackend)
|
||||
stored_at = await backend.upload_file(file_content=b"line\n", filename="input.jsonl", content_type="text/plain")
|
||||
assert stored_at == f"{LITELLM_DB_STORAGE_URL_PREFIX}row-1"
|
||||
table.create.assert_awaited_once()
|
||||
|
||||
|
||||
def test_litellm_db_backend_without_a_database_is_rejected():
|
||||
with pytest.raises(ValueError, match="database-connected proxy"):
|
||||
get_storage_backend(LITELLM_DB_STORAGE_BACKEND_NAME)
|
||||
|
||||
|
||||
def test_unknown_backend_is_still_rejected():
|
||||
with pytest.raises(ValueError, match="Unsupported storage backend type: nope"):
|
||||
get_storage_backend("nope", prisma_client=MagicMock())
|
||||
|
|
@ -1552,7 +1552,7 @@ async def test_anthropic_post_uses_prebuilt_body_without_redumping():
|
|||
provider_config = Mock()
|
||||
provider_config.max_retry_on_anthropic_messages_http_error = 2
|
||||
|
||||
logging_obj = Mock()
|
||||
logging_obj: Final = Mock(baseline_cache_context=None)
|
||||
logging_obj.model_call_details = {}
|
||||
|
||||
out = await handler._async_post_anthropic_messages_with_http_error_retry(
|
||||
|
|
@ -1592,7 +1592,7 @@ async def test_anthropic_post_falls_back_to_json_dumps_when_unsigned_none():
|
|||
|
||||
provider_config = Mock()
|
||||
provider_config.max_retry_on_anthropic_messages_http_error = 1
|
||||
logging_obj = Mock()
|
||||
logging_obj: Final = Mock(baseline_cache_context=None)
|
||||
logging_obj.model_call_details = {}
|
||||
|
||||
await handler._async_post_anthropic_messages_with_http_error_retry(
|
||||
|
|
@ -1640,7 +1640,7 @@ async def test_anthropic_post_retry_reserializes_mutated_body():
|
|||
# Re-sign returns no signed body (native anthropic path) -> must re-dump.
|
||||
provider_config.sign_request = Mock(return_value=({}, None))
|
||||
|
||||
logging_obj = Mock()
|
||||
logging_obj: Final = Mock(baseline_cache_context=None)
|
||||
logging_obj.model_call_details = {}
|
||||
|
||||
await handler._async_post_anthropic_messages_with_http_error_retry(
|
||||
|
|
@ -2579,7 +2579,7 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques
|
|||
posts.append({"headers": dict(headers), "data": data})
|
||||
return invalid_signature_response if len(posts) == 1 else ok_response
|
||||
|
||||
logging_obj = Mock()
|
||||
logging_obj: Final = Mock(baseline_cache_context=None)
|
||||
logging_obj.model_call_details = {}
|
||||
|
||||
response = await handler._async_post_anthropic_messages_with_http_error_retry(
|
||||
|
|
|
|||
|
|
@ -629,7 +629,7 @@ class TestManagedTables:
|
|||
|
||||
class TestAutoRouterSession:
|
||||
@staticmethod
|
||||
def _row(baseline_models: dict) -> LiteLLM_AutoRouterSession:
|
||||
def _row(estimated_baseline_models: dict[str, int]) -> LiteLLM_AutoRouterSession:
|
||||
return LiteLLM_AutoRouterSession(
|
||||
api_key="k",
|
||||
session_id="s",
|
||||
|
|
@ -643,7 +643,9 @@ class TestAutoRouterSession:
|
|||
saved_spend=0.24,
|
||||
classifier_cost=0.0,
|
||||
tier_turns={},
|
||||
baseline_models=baseline_models,
|
||||
baseline_models={"legacy-baseline": 100},
|
||||
savings_estimated_turns=sum(estimated_baseline_models.values()),
|
||||
savings_estimated_baseline_models=estimated_baseline_models,
|
||||
)
|
||||
|
||||
def test_the_baseline_label_is_the_one_most_turns_were_priced_against(self):
|
||||
|
|
@ -655,5 +657,5 @@ class TestAutoRouterSession:
|
|||
assert self._row({"b-model": 1, "a-model": 1}).baseline_model == "b-model"
|
||||
assert self._row({"a-model": 1, "b-model": 1}).baseline_model == "b-model"
|
||||
|
||||
def test_a_row_whose_turns_recorded_no_baseline_has_no_label(self):
|
||||
def test_a_row_without_current_estimates_has_no_baseline_label(self) -> None:
|
||||
assert self._row({}).baseline_model is None
|
||||
|
|
|
|||
|
|
@ -5739,9 +5739,15 @@ class TestMCPDcrBridgeDelegateAdmission:
|
|||
prisma and are swallowed (``_safe_fetch`` / the SCIM gate's fail-open), so their checks
|
||||
skip. Yields the ``get_key_object`` mock so callers can assert the sealed ``key_hash`` was
|
||||
the reload key."""
|
||||
from litellm.proxy.auth.auth_checks import OrganizationNotFoundError
|
||||
|
||||
get_key_object = AsyncMock(return_value=return_value, side_effect=side_effect)
|
||||
get_org_object = AsyncMock(side_effect=OrganizationNotFoundError("Organization doesn't exist in db."))
|
||||
patchers = [
|
||||
patch("litellm.proxy.auth.auth_checks.get_key_object", get_key_object),
|
||||
patch( # test-quality-ok: central auth now resolves org limits; this fixture models a missing org row
|
||||
"litellm.proxy.auth.auth_checks.get_org_object", get_org_object
|
||||
),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -11974,9 +11974,13 @@ async def test_oauth_credential_write_keeps_virtual_key_permissions(
|
|||
from litellm.proxy._experimental.mcp_server import mcp_server_manager
|
||||
from litellm.proxy._experimental.mcp_server.bridge_token_flow import authorize_oauth_credential_request
|
||||
from litellm.proxy._types import UserAPIKeyAuth, hash_token
|
||||
from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key
|
||||
from litellm.proxy.auth.auth_checks import OrganizationNotFoundError, jwt_key_mapping_cache_key
|
||||
|
||||
handler, signing_key = jwt_oauth_identity
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.auth.auth_checks.get_org_object",
|
||||
AsyncMock(side_effect=OrganizationNotFoundError("Organization doesn't exist in db.")),
|
||||
)
|
||||
key: Final = "sk-oauth-permission-test"
|
||||
hashed: Final = hash_token(key)
|
||||
credential: Final = UserAPIKeyAuth(
|
||||
|
|
|
|||
|
|
@ -6087,6 +6087,107 @@ async def test_organization_budget_check_carries_org_state_on_the_token():
|
|||
assert token.org_budget_snapshot == OrgBudgetSnapshot(spend=12.5, max_budget=100.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("warmed_by_auth_prefetch", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_org_object_for_request_serves_last_known_org_through_db_outage(warmed_by_auth_prefetch):
|
||||
"""A JWT whose team sits in an org resolves the org on every request, and the org row
|
||||
is cached for only DEFAULT_IN_MEMORY_TTL seconds while the team and user rows ride the
|
||||
60s management-object TTL. Without a last-known copy, a DB outage a few seconds old
|
||||
turned that traffic into 503s while the same request through a virtual key kept
|
||||
succeeding on its cached team. The copy must exist whoever filled the short-lived entry:
|
||||
this lookup's own DB read, or the virtual-key auth prefetch warming it for the same org."""
|
||||
from litellm.proxy._types import LiteLLM_OrganizationTable
|
||||
from litellm.proxy.auth.auth_checks import get_org_object_for_request
|
||||
|
||||
org_columns = {
|
||||
"organization_id": "org-1",
|
||||
"organization_alias": "platform-org",
|
||||
"budget_id": "b1",
|
||||
"created_by": "admin",
|
||||
"updated_by": "admin",
|
||||
"litellm_budget_table": {"budget_id": "b1", "max_budget": 50.0, "tpm_limit": 700, "rpm_limit": 7},
|
||||
}
|
||||
org_row = MagicMock()
|
||||
org_row.model_dump = lambda: org_columns
|
||||
db_outage = ConnectionRefusedError("db unavailable")
|
||||
prisma_client = MagicMock()
|
||||
prisma_client.db.litellm_organizationtable.find_unique = AsyncMock(
|
||||
side_effect=[db_outage] if warmed_by_auth_prefetch else [org_row, db_outage]
|
||||
)
|
||||
user_api_key_cache = UserApiKeyCache()
|
||||
if warmed_by_auth_prefetch:
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="org_id:org-1:with_budget",
|
||||
value=LiteLLM_OrganizationTable.model_validate(org_columns),
|
||||
model_type=LiteLLM_OrganizationTable,
|
||||
)
|
||||
|
||||
async def _lookup():
|
||||
return await get_org_object_for_request(
|
||||
org_id="org-1",
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=None,
|
||||
proxy_logging_obj=None,
|
||||
)
|
||||
|
||||
with patch("litellm.proxy.proxy_server.general_settings", {}): # test-quality-ok: the outage fallback reads this module global; no dependency injection seam exists
|
||||
warm = await _lookup()
|
||||
assert warm is not None and warm.organization_alias == "platform-org"
|
||||
await user_api_key_cache.async_delete_cache("org_id:org-1:with_budget")
|
||||
|
||||
during_outage = await _lookup()
|
||||
|
||||
assert prisma_client.db.litellm_organizationtable.find_unique.await_count == (1 if warmed_by_auth_prefetch else 2)
|
||||
assert during_outage is not None
|
||||
assert during_outage.organization_alias == "platform-org"
|
||||
assert during_outage.litellm_budget_table is not None
|
||||
assert during_outage.litellm_budget_table.rpm_limit == 7
|
||||
assert during_outage.litellm_budget_table.max_budget == 50.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_org_object_for_request_writes_the_last_known_org_only_when_absent():
|
||||
"""The last-known copy is written when this worker holds none, never per request:
|
||||
with Redis attached, a write on every cached org hit would cost one SET per JWT request."""
|
||||
from litellm.proxy._types import LiteLLM_OrganizationTable
|
||||
from litellm.proxy.auth.auth_checks import get_org_object_for_request
|
||||
|
||||
class _WriteRecordingCache(UserApiKeyCache):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.written_keys = []
|
||||
|
||||
async def async_set_cache(self, key, value, local_only=False, **kwargs):
|
||||
self.written_keys.append(key)
|
||||
return await super().async_set_cache(key=key, value=value, local_only=local_only, **kwargs)
|
||||
|
||||
user_api_key_cache = _WriteRecordingCache()
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="org_id:org-1:with_budget",
|
||||
value=LiteLLM_OrganizationTable(
|
||||
organization_id="org-1",
|
||||
organization_alias="platform-org",
|
||||
budget_id="b1",
|
||||
created_by="admin",
|
||||
updated_by="admin",
|
||||
),
|
||||
model_type=LiteLLM_OrganizationTable,
|
||||
)
|
||||
|
||||
for _ in range(3):
|
||||
org = await get_org_object_for_request(
|
||||
org_id="org-1",
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=None,
|
||||
proxy_logging_obj=None,
|
||||
)
|
||||
assert org is not None and org.organization_alias == "platform-org"
|
||||
|
||||
assert user_api_key_cache.written_keys.count("org_id:org-1:with_budget:last_known") == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"max_budget, spend, expect_blocked",
|
||||
[
|
||||
|
|
@ -7572,7 +7673,7 @@ async def test_project_allowlist_enforced_when_key_models_empty():
|
|||
assert exc_info.value.code == "403"
|
||||
|
||||
|
||||
def _project_with_budget(spend: float, max_budget: float):
|
||||
def _project_with_budget(spend: float, max_budget: float | None):
|
||||
from litellm.proxy._types import LiteLLM_BudgetTable, LiteLLM_ProjectTableCachedObj
|
||||
|
||||
return LiteLLM_ProjectTableCachedObj(
|
||||
|
|
@ -7592,11 +7693,12 @@ def _project_with_budget(spend: float, max_budget: float):
|
|||
pytest.param(4.99, 0.0, 5.0, False, id="counter-under-budget-admits"),
|
||||
pytest.param(None, 5.0, 5.0, True, id="no-counter-falls-back-to-persisted-spend"),
|
||||
pytest.param(None, 0.0, 5.0, False, id="no-counter-and-no-persisted-spend-admits"),
|
||||
pytest.param(12.5, 12.5, 0.0, False, id="zero-budget-is-unbudgeted"),
|
||||
pytest.param(12.5, 12.5, -1.0, False, id="negative-budget-is-unbudgeted"),
|
||||
pytest.param(None, 0.0, 0.0, True, id="zero-budget-blocks-before-any-spend"),
|
||||
pytest.param(12.5, 12.5, 0.0, True, id="zero-budget-blocks-with-spend"),
|
||||
pytest.param(12.5, 12.5, None, False, id="null-budget-is-unlimited"),
|
||||
],
|
||||
)
|
||||
async def test_project_max_budget_check_blocks_only_when_live_spend_reaches_a_positive_budget(
|
||||
async def test_project_max_budget_check_blocks_when_live_spend_reaches_the_budget(
|
||||
counter_spend, db_spend, max_budget, blocks
|
||||
):
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
|
|
@ -7631,7 +7733,7 @@ async def test_project_max_budget_check_blocks_only_when_live_spend_reaches_a_po
|
|||
|
||||
assert exc_info.value.entity_type == Litellm_EntityType.PROJECT.value
|
||||
assert exc_info.value.entity_id == "p-budget"
|
||||
assert exc_info.value.current_cost == 5.0
|
||||
assert exc_info.value.current_cost == (db_spend if counter_spend is None else counter_spend)
|
||||
proxy_logging_obj.budget_alerts.assert_awaited_once()
|
||||
assert proxy_logging_obj.budget_alerts.await_args.kwargs["type"] == "project_budget"
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.proxy._types import (
|
|||
LiteLLM_JWTAuth,
|
||||
LiteLLM_BudgetTable,
|
||||
LiteLLM_EndUserTable,
|
||||
LiteLLM_OrganizationTable,
|
||||
LiteLLM_TeamTableCachedObj,
|
||||
LiteLLM_UserTable,
|
||||
LitellmUserRoles,
|
||||
|
|
@ -35,6 +36,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
OrganizationNotFoundError,
|
||||
TeamNotFoundError,
|
||||
UserNotFoundError,
|
||||
get_key_object,
|
||||
|
|
@ -5986,6 +5988,152 @@ async def test_centralized_common_checks_backfills_org_id_from_team(key_org_id,
|
|||
setattr(_proxy_server_mod, k, v)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"key_org_id,team_id,team_org_id,existing_alias,existing_rpm,lookup_mode,allow_db_unavailable,expect_lookup_error,expected_org_id,expected_alias,expected_limits",
|
||||
[
|
||||
(None, "t1", "org-from-team", None, None, "success", False, False, "org-from-team", "acme-org", (12.5, 700, 7)),
|
||||
("org-jwt", None, None, None, None, "success", False, False, "org-jwt", "acme-org", (12.5, 700, 7)),
|
||||
("org-pinned", None, None, "preset", None, "success", False, False, "org-pinned", "preset", (None, None, None)),
|
||||
("org-view", None, None, None, 3, "success", False, False, "org-view", None, (None, None, 3)),
|
||||
("org-missing", None, None, None, None, "missing", False, False, "org-missing", None, (None, None, None)),
|
||||
("org-db-failure-allowed", None, None, None, None, "db_failure", True, False, "org-db-failure-allowed", None, (None, None, None)),
|
||||
("org-db-failure-denied", None, None, None, None, "db_failure", False, True, "org-db-failure-denied", None, (None, None, None)),
|
||||
("org-bad-row", None, None, None, None, "bad_row", False, False, "org-bad-row", None, (None, None, None)),
|
||||
("org-nobudget", None, None, None, None, "no_budget", False, False, "org-nobudget", "acme-org", (None, None, None)),
|
||||
],
|
||||
)
|
||||
async def test_centralized_common_checks_inherits_org_identity(
|
||||
key_org_id: str | None,
|
||||
team_id: str | None,
|
||||
team_org_id: str | None,
|
||||
existing_alias: str | None,
|
||||
existing_rpm: int | None,
|
||||
lookup_mode: str,
|
||||
allow_db_unavailable: bool,
|
||||
expect_lookup_error: bool,
|
||||
expected_org_id: str | None,
|
||||
expected_alias: str | None,
|
||||
expected_limits: tuple[float | None, int | None, int | None],
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from fastapi import Request
|
||||
from starlette.datastructures import URL
|
||||
|
||||
from litellm.proxy._types import LiteLLM_TeamTableCachedObj
|
||||
|
||||
token = UserAPIKeyAuth(
|
||||
api_key="sk-test",
|
||||
user_id="u",
|
||||
team_id=team_id,
|
||||
org_id=key_org_id,
|
||||
organization_alias=existing_alias,
|
||||
organization_rpm_limit=existing_rpm,
|
||||
)
|
||||
request = Request(scope={"type": "http"})
|
||||
request._url = URL(url="/chat/completions")
|
||||
|
||||
fetched_team = (
|
||||
LiteLLM_TeamTableCachedObj(team_id="t1", organization_id=team_org_id) if team_id is not None else None
|
||||
)
|
||||
organization = LiteLLM_OrganizationTable(
|
||||
organization_id=expected_org_id,
|
||||
organization_alias="acme-org",
|
||||
budget_id="budget-id",
|
||||
metadata={"model_rpm_limit": {"gpt-4o": 2}},
|
||||
models=[],
|
||||
created_by="test",
|
||||
updated_by="test",
|
||||
litellm_budget_table=(
|
||||
None
|
||||
if lookup_mode == "no_budget"
|
||||
else LiteLLM_BudgetTable(budget_id="budget-id", max_budget=12.5, tpm_limit=700, rpm_limit=7)
|
||||
),
|
||||
)
|
||||
|
||||
attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None)
|
||||
attrs["prisma_client"] = MagicMock()
|
||||
attrs["general_settings"] = {"allow_requests_on_db_unavailable": allow_db_unavailable}
|
||||
originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs}
|
||||
try:
|
||||
for k, v in attrs.items():
|
||||
setattr(_proxy_server_mod, k, v)
|
||||
with (
|
||||
patch( # test-quality-ok: centralized auth calls this module helper directly; no dependency injection seam exists
|
||||
"litellm.proxy.auth.user_api_key_auth.get_team_object",
|
||||
new_callable=AsyncMock,
|
||||
return_value=fetched_team,
|
||||
) as mock_get_team_object,
|
||||
patch( # test-quality-ok: centralized auth calls this module helper directly; no dependency injection seam exists
|
||||
"litellm.proxy.auth.auth_checks.get_org_object",
|
||||
new_callable=AsyncMock,
|
||||
return_value=organization,
|
||||
) as mock_get_org_object,
|
||||
patch( # test-quality-ok: capture downstream token state without invoking unrelated common checks
|
||||
"litellm.proxy.auth.user_api_key_auth.common_checks",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_checks,
|
||||
):
|
||||
if lookup_mode == "missing":
|
||||
mock_get_org_object.side_effect = OrganizationNotFoundError("x")
|
||||
elif lookup_mode == "db_failure":
|
||||
mock_get_org_object.side_effect = ConnectionRefusedError("db unavailable")
|
||||
elif lookup_mode == "bad_row":
|
||||
mock_get_org_object.side_effect = ValueError("row failed validation")
|
||||
|
||||
if expect_lookup_error:
|
||||
with pytest.raises(ConnectionRefusedError, match="db unavailable"):
|
||||
await _run_centralized_common_checks(
|
||||
user_api_key_auth_obj=token,
|
||||
request=request,
|
||||
request_data={"model": "gpt-4o"},
|
||||
route="/chat/completions",
|
||||
)
|
||||
else:
|
||||
await _run_centralized_common_checks(
|
||||
user_api_key_auth_obj=token,
|
||||
request=request,
|
||||
request_data={"model": "gpt-4o"},
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert token.org_id == expected_org_id
|
||||
if expect_lookup_error:
|
||||
mock_checks.assert_not_awaited()
|
||||
assert token.organization_alias is None
|
||||
assert token.organization_max_budget is None
|
||||
assert token.organization_tpm_limit is None
|
||||
assert token.organization_rpm_limit is None
|
||||
return
|
||||
|
||||
mock_checks.assert_awaited_once()
|
||||
assert token.organization_alias == expected_alias
|
||||
assert (
|
||||
token.organization_max_budget,
|
||||
token.organization_tpm_limit,
|
||||
token.organization_rpm_limit,
|
||||
) == expected_limits
|
||||
checked_token = mock_checks.await_args.kwargs["valid_token"]
|
||||
assert checked_token.org_id == expected_org_id
|
||||
assert checked_token.organization_alias == expected_alias
|
||||
if team_id is None:
|
||||
mock_get_team_object.assert_not_awaited()
|
||||
else:
|
||||
mock_get_team_object.assert_awaited_once()
|
||||
if existing_alias is not None or existing_rpm is not None:
|
||||
mock_get_org_object.assert_not_awaited()
|
||||
assert token.organization_metadata is None
|
||||
else:
|
||||
mock_get_org_object.assert_awaited_once()
|
||||
assert mock_get_org_object.await_args.kwargs["org_id"] == expected_org_id
|
||||
assert mock_get_org_object.await_args.kwargs["include_budget_table"] is True
|
||||
if lookup_mode not in {"missing", "db_failure", "bad_row"}:
|
||||
assert token.organization_metadata == {"model_rpm_limit": {"gpt-4o": 2}}
|
||||
finally:
|
||||
for k, v in originals.items():
|
||||
setattr(_proxy_server_mod, k, v)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cli_session_token_org_backfilled_from_team(monkeypatch):
|
||||
"""LIT-4688 root cause: CLI session tokens (from /sso/cli/poll) are minted
|
||||
|
|
|
|||
|
|
@ -34,10 +34,13 @@ import json
|
|||
import logging
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
|
||||
import litellm
|
||||
|
|
@ -52,7 +55,7 @@ from litellm.proxy.utils import ProxyLogging
|
|||
from litellm.router import Router
|
||||
from litellm.types.llms.openai import BatchJobStatus
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import CredentialItem, LiteLLMBatch
|
||||
from litellm.types.utils import CredentialItem, LiteLLMBatch, SpecialEnums
|
||||
|
||||
from fastapi import Request, Response
|
||||
|
||||
|
|
@ -74,6 +77,12 @@ CREDS: Dict[str, Dict[str, str]] = {
|
|||
"api_base": "https://vertex.test",
|
||||
"model": "vertex_ai/gemini-2.0",
|
||||
},
|
||||
"my-vllm": {
|
||||
"custom_llm_provider": "hosted_vllm",
|
||||
"api_key": "sk-vllm",
|
||||
"api_base": "http://vllm.test/v1",
|
||||
"model": "hosted_vllm/qwen",
|
||||
},
|
||||
}
|
||||
|
||||
# A real model-encoded file id: decodes to "azure/gpt-4o", strips to "file-original123".
|
||||
|
|
@ -147,6 +156,7 @@ class Harness:
|
|||
router: MagicMock
|
||||
logging: MagicMock
|
||||
creds_resolver: MagicMock
|
||||
upstream_files_route: respx.Route
|
||||
|
||||
@property
|
||||
def router_acreate(self) -> AsyncMock:
|
||||
|
|
@ -162,13 +172,14 @@ class Harness:
|
|||
return dict(self.router_acreate.call_args.kwargs)
|
||||
|
||||
|
||||
def _creds_lookup(*, model_id: str) -> Dict[str, str]:
|
||||
# KeyError on an unknown/hardcoded model_id - the bug cannot hide.
|
||||
return dict(CREDS[model_id])
|
||||
def _creds_lookup(*, model_id: str, team_id: str | None = None) -> dict[str, str] | None:
|
||||
# An unknown/hardcoded model_id resolves to None exactly like the real router,
|
||||
# which the endpoint turns into a 400 and a missing dispatch - the bug cannot hide.
|
||||
return dict(CREDS[model_id]) if model_id in CREDS else None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def harness():
|
||||
def harness(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Seam harness. Patches only true I/O boundaries; pure encode/decode/merge
|
||||
helpers run for real. Object mocks are spec'd so unknown method calls raise."""
|
||||
body_holder: Dict[str, Any] = {}
|
||||
|
|
@ -192,6 +203,7 @@ def harness():
|
|||
provider_from_headers = MagicMock(return_value=None)
|
||||
is_known_model = MagicMock(return_value=False)
|
||||
litellm_acreate = AsyncMock(return_value=make_batch())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(endpoints, "_read_request_body", read_body))
|
||||
|
|
@ -213,6 +225,10 @@ def harness():
|
|||
stack.enter_context(patch.object(endpoints, "is_known_model", is_known_model))
|
||||
stack.enter_context(patch.object(litellm, "acreate_batch", litellm_acreate))
|
||||
stack.enter_context(patch.object(litellm, "enable_loadbalancing_on_batch_endpoints", False))
|
||||
upstream = stack.enter_context(respx.mock(assert_all_called=False))
|
||||
upstream_files_route = upstream.get(f"{CREDS['my-vllm']['api_base']}/files").mock(
|
||||
return_value=httpx.Response(404, json={"detail": "Not Found"})
|
||||
)
|
||||
stack.enter_context(patch.object(proxy_server, "llm_router", router))
|
||||
stack.enter_context(patch.object(proxy_server, "proxy_logging_obj", logging))
|
||||
stack.enter_context(patch.object(proxy_server, "general_settings", {}))
|
||||
|
|
@ -231,6 +247,7 @@ def harness():
|
|||
router=router,
|
||||
logging=logging,
|
||||
creds_resolver=router.get_deployment_credentials_with_provider,
|
||||
upstream_files_route=upstream_files_route,
|
||||
)
|
||||
yield h
|
||||
|
||||
|
|
@ -255,6 +272,25 @@ async def call_create(
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def executed_runner():
|
||||
runner = MagicMock(spec=endpoints.LiteLLMExecutedBatchRunner)
|
||||
runner.create = AsyncMock(return_value=make_batch(id="litellm-executed-batch"))
|
||||
runner.cancel = AsyncMock(return_value=make_batch(id="litellm-executed-batch", status="cancelling"))
|
||||
factory = MagicMock(return_value=runner)
|
||||
with patch.object( # test-quality-ok: the route builds its runner from proxy_server globals; the factory is the only seam
|
||||
endpoints, "_litellm_executed_batch_runner", factory
|
||||
):
|
||||
yield runner, factory
|
||||
|
||||
|
||||
def _managed_input_file_id(model: str) -> str:
|
||||
unified = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
|
||||
"application/jsonl", "managed-id", model, "file-id", "file-model-id"
|
||||
)
|
||||
return base64.urlsafe_b64encode(unified.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
# SCENARIO 1 - input_file_id encoded with model. The full showcase: every
|
||||
# assertion type from the design lives here.
|
||||
|
|
@ -766,6 +802,137 @@ async def test_create__unified_file_id_legacy_row_without_storage_url_dispatches
|
|||
assert harness.router_kwargs()["input_file_id"] == "litellm_proxy_unified_id"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# LiteLLM-executed batches: a unified file targeting a provider whose API has
|
||||
# no /v1/batches (hosted_vllm) runs inside LiteLLM instead of being forwarded.
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__unified_executed_provider_runs_inside_litellm(harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
caller = UserAPIKeyAuth(api_key="sk-test", team_id="team-vllm")
|
||||
input_file_id = _managed_input_file_id("my-vllm")
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": input_file_id,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"litellm_metadata": {"tags": ["batch-tag"]},
|
||||
},
|
||||
)
|
||||
resp = await call_create(harness, user=caller)
|
||||
|
||||
harness.router_acreate.assert_not_called()
|
||||
harness.litellm_acreate.assert_not_called()
|
||||
harness.creds_resolver.assert_called_once_with(model_id="my-vllm", team_id="team-vllm")
|
||||
factory.assert_called_once_with(harness.router, harness.logging)
|
||||
runner.create.assert_awaited_once()
|
||||
create_kwargs = runner.create.call_args.kwargs
|
||||
assert create_kwargs["unified_input_file_id"] == input_file_id
|
||||
assert create_kwargs["model"] == "my-vllm"
|
||||
assert create_kwargs["provider"] == "hosted_vllm"
|
||||
assert create_kwargs["request_tags"] == ("batch-tag",)
|
||||
assert create_kwargs["user_api_key_dict"] is caller
|
||||
assert create_kwargs["create_request"]["model"] == "my-vllm"
|
||||
assert resp.id == "litellm-executed-batch"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__unified_executed_provider_without_database_400(harness):
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": _managed_input_file_id("my-vllm"),
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
},
|
||||
)
|
||||
with pytest.raises(ProxyException) as exc:
|
||||
await call_create(harness)
|
||||
|
||||
assert exc.value.code == "400"
|
||||
assert "need a database" in exc.value.message
|
||||
harness.router_acreate.assert_not_called()
|
||||
harness.litellm_acreate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__unified_executed_provider_with_its_own_files_api_goes_to_the_provider(harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
harness.upstream_files_route.mock(return_value=httpx.Response(200, json={"object": "list", "data": []}))
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": _managed_input_file_id("my-vllm"),
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
},
|
||||
)
|
||||
await call_create(harness)
|
||||
|
||||
factory.assert_not_called()
|
||||
runner.create.assert_not_called()
|
||||
assert harness.router_kwargs()["model"] == "my-vllm"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__unified_provider_model_never_touches_executed_runner(harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": _managed_input_file_id("azure/gpt-4o"),
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
},
|
||||
)
|
||||
await call_create(harness)
|
||||
|
||||
factory.assert_not_called()
|
||||
runner.create.assert_not_called()
|
||||
harness.creds_resolver.assert_called_once_with(model_id="azure/gpt-4o", team_id=None)
|
||||
assert harness.router_kwargs()["model"] == "azure/gpt-4o"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("via", ["body", "header"])
|
||||
async def test_create__raw_file_with_executed_model_400_with_upload_guidance(harness, via):
|
||||
body = {"input_file_id": "file-plain", "endpoint": "/v1/chat/completions", "completion_window": "24h"}
|
||||
set_body(harness, {**body, "model": "my-vllm"} if via == "body" else body)
|
||||
headers = {"x-litellm-model": "my-vllm"} if via == "header" else None
|
||||
|
||||
with pytest.raises(ProxyException) as exc:
|
||||
await call_create(harness, headers=headers)
|
||||
|
||||
assert exc.value.code == "400"
|
||||
assert "POST /v1/files" in exc.value.message
|
||||
assert "x-litellm-model" in exc.value.message
|
||||
harness.litellm_acreate.assert_not_called()
|
||||
harness.router_acreate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"upstream_answer",
|
||||
[httpx.Response(200, json={"object": "list", "data": []}), httpx.Response(405), httpx.ConnectError("refused")],
|
||||
ids=["lists files", "files route without list", "unreachable"],
|
||||
)
|
||||
async def test_create__raw_file_with_executed_model_is_forwarded_unless_the_server_lacks_a_files_api(
|
||||
harness, upstream_answer
|
||||
):
|
||||
harness.upstream_files_route.mock(side_effect=[upstream_answer])
|
||||
set_body(harness, {"input_file_id": "file-plain", "endpoint": "/v1/chat/completions", "completion_window": "24h"})
|
||||
|
||||
await call_create(harness, headers={"x-litellm-model": "my-vllm"})
|
||||
|
||||
forwarded = harness.acreate_kwargs()
|
||||
assert forwarded["input_file_id"] == "file-plain"
|
||||
assert forwarded["custom_llm_provider"] == "hosted_vllm"
|
||||
assert forwarded["api_base"] == CREDS["my-vllm"]["api_base"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__model_encoded_beats_unified(harness):
|
||||
"""Precedence row: a file id that is BOTH model-encoded and (pretend) unified
|
||||
|
|
@ -1146,6 +1313,11 @@ AZURE_BATCH_ID = encode_file_id_with_model("batch_orig123", "azure/gpt-4o", id_t
|
|||
# returns). model_id / llm_batch_id are parsed out of this by the real helpers.
|
||||
UNIFIED_BATCH_ID = "litellm_proxy;model_id:gpt-4o-mini;llm_batch_id:batch-raw-xyz"
|
||||
|
||||
# A decoded unified id of a batch LiteLLM runs itself: the llm_batch_id carries
|
||||
# the litellm_batch_ prefix, so no provider holds a batch to sync with.
|
||||
EXECUTED_BATCH_ID = "litellm_proxy;model_id:my-vllm;llm_batch_id:litellm_batch_abc"
|
||||
EXECUTED_BATCH_B64 = base64.urlsafe_b64encode(EXECUTED_BATCH_ID.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RetrieveHarness:
|
||||
|
|
@ -1580,6 +1752,69 @@ async def test_retrieve__db_non_terminal_state_syncs_with_provider(retrieve_harn
|
|||
assert retrieve_harness.update_batch_in_db.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["validating", "in_progress", "finalizing", "cancelling"])
|
||||
async def test_retrieve__executed_batch_served_from_db_in_every_status(retrieve_harness, status):
|
||||
db_response = make_batch(id="litellm-executed-batch", status=status)
|
||||
db_batch_object = MagicMock()
|
||||
db_batch_object.updated_at = datetime.now(timezone.utc)
|
||||
retrieve_harness.get_batch_from_db.return_value = (db_batch_object, db_response)
|
||||
|
||||
resp = await call_retrieve(retrieve_harness, EXECUTED_BATCH_B64)
|
||||
|
||||
assert resp is db_response
|
||||
retrieve_harness.litellm_aretrieve.assert_not_called()
|
||||
retrieve_harness.router_aretrieve.assert_not_called()
|
||||
retrieve_harness.update_batch_in_db.assert_not_called()
|
||||
retrieve_harness.ensure_managed_files.assert_called_once()
|
||||
assert retrieve_harness.ensure_managed_files.call_args.kwargs["unified_batch_id"] == EXECUTED_BATCH_ID
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve__executed_batch_abandoned_by_its_runner_is_served_failed(retrieve_harness, executed_runner):
|
||||
runner, _ = executed_runner
|
||||
failed = make_batch(id="litellm-executed-batch", status="failed")
|
||||
runner.fail_abandoned = AsyncMock(return_value=failed)
|
||||
db_response = make_batch(id="litellm-executed-batch", status="in_progress")
|
||||
db_batch_object = MagicMock()
|
||||
db_batch_object.updated_at = datetime.now(timezone.utc) - timedelta(minutes=10)
|
||||
retrieve_harness.get_batch_from_db.return_value = (db_batch_object, db_response)
|
||||
user = UserAPIKeyAuth(api_key="sk-test", user_id="user-1")
|
||||
|
||||
resp = await call_retrieve(retrieve_harness, EXECUTED_BATCH_B64, user=user)
|
||||
|
||||
assert resp is failed
|
||||
runner.fail_abandoned.assert_awaited_once_with(db_response, user)
|
||||
retrieve_harness.litellm_aretrieve.assert_not_called()
|
||||
retrieve_harness.router_aretrieve.assert_not_called()
|
||||
retrieve_harness.ensure_managed_files.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve__executed_batch_with_a_fresh_heartbeat_is_left_running(retrieve_harness, executed_runner):
|
||||
runner, _ = executed_runner
|
||||
runner.fail_abandoned = AsyncMock()
|
||||
db_response = make_batch(id="litellm-executed-batch", status="in_progress")
|
||||
db_batch_object = MagicMock()
|
||||
db_batch_object.updated_at = datetime.now(timezone.utc) - timedelta(seconds=30)
|
||||
retrieve_harness.get_batch_from_db.return_value = (db_batch_object, db_response)
|
||||
|
||||
resp = await call_retrieve(retrieve_harness, EXECUTED_BATCH_B64)
|
||||
|
||||
assert resp is db_response
|
||||
runner.fail_abandoned.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve__executed_batch_without_db_row_404(retrieve_harness):
|
||||
with pytest.raises(ProxyException) as exc:
|
||||
await call_retrieve(retrieve_harness, EXECUTED_BATCH_B64)
|
||||
|
||||
assert exc.value.code == "404"
|
||||
retrieve_harness.litellm_aretrieve.assert_not_called()
|
||||
retrieve_harness.router_aretrieve.assert_not_called()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cross-cutting: enrichment route_type and failure-hook on provider error.
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -2299,6 +2534,35 @@ async def test_cancel__unified_no_router_500(cancel_harness):
|
|||
assert exc.value.code == "500"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel__executed_batch_routes_to_runner(cancel_harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
caller = UserAPIKeyAuth(api_key="sk-test", user_id="user-cancel-2")
|
||||
resp = await call_cancel(cancel_harness, EXECUTED_BATCH_B64, user=caller)
|
||||
|
||||
runner.cancel.assert_awaited_once_with(EXECUTED_BATCH_B64, caller)
|
||||
factory.assert_called_once_with(cancel_harness.router, cancel_harness.logging)
|
||||
cancel_harness.router_acancel.assert_not_called()
|
||||
cancel_harness.litellm_acancel.assert_not_called()
|
||||
cancel_harness.creds_resolver.assert_not_called()
|
||||
assert resp is runner.cancel.return_value
|
||||
assert cancel_harness.update_batch_in_db.call_args.kwargs["operation"] == "cancel"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel__executed_batch_no_router_500(cancel_harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
with patch.object( # test-quality-ok: proxy_server module global is the endpoint's only injection point
|
||||
proxy_server, "llm_router", None
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc:
|
||||
await call_cancel(cancel_harness, EXECUTED_BATCH_B64)
|
||||
|
||||
assert exc.value.code == "500"
|
||||
factory.assert_not_called()
|
||||
runner.cancel.assert_not_called()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SCENARIO 3 - fallback to custom_llm_provider. Rebuilds a CancelBatchRequest
|
||||
# and forwards only {custom_llm_provider, batch_id}.
|
||||
|
|
@ -2956,3 +3220,16 @@ async def test_cancel__unified_batch_id_rejects_key_without_model_grant(cancel_h
|
|||
|
||||
assert exc_info.value.code == "403"
|
||||
cancel_harness.router_acancel.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel__executed_batch_rejects_key_without_model_grant(cancel_harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await call_cancel(cancel_harness, EXECUTED_BATCH_B64, user=_key_restricted_to("vertex-model"))
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
factory.assert_not_called()
|
||||
runner.cancel.assert_not_called()
|
||||
cancel_harness.router_acancel.assert_not_called()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -325,9 +325,12 @@ class TestRender:
|
|||
use_color=False,
|
||||
)
|
||||
|
||||
def test_a_session_that_cost_more_than_its_baseline_reads_as_a_plus(self, config_dir):
|
||||
dearer = RECORDED._replace(spend=0.50, baseline_spend=0.40)
|
||||
assert "+25% vs Claude Opus 5" in render("m", dearer, config_dir, use_color=False)
|
||||
@pytest.mark.parametrize("spend,delta", ((0.50, "+25%"), (0.40, "0%"), (0.4001, "0%"), (0.3999, "0%"), (0.30, "-25%")))
|
||||
def test_rounded_cost_delta_uses_a_sign_only_for_nonzero_percentages(
|
||||
self, config_dir: Path, spend: float, delta: str,
|
||||
) -> None:
|
||||
session: Final = RECORDED._replace(spend=spend, baseline_spend=0.40)
|
||||
assert render("m", session, config_dir, use_color=False).splitlines()[0] == f"Routed to: m {delta} vs Claude Opus 5"
|
||||
|
||||
def test_without_a_baseline_only_the_routed_line_shows(self, config_dir):
|
||||
assert render("m", RECORDED._replace(baseline_model=None), config_dir, False) == "Routed to: m"
|
||||
|
|
@ -339,6 +342,37 @@ class TestRender:
|
|||
|
||||
|
||||
class TestClaudeCodeMode:
|
||||
@pytest.mark.parametrize("estimated_turns", (0, 1))
|
||||
def test_current_estimates_keep_the_routed_model_and_compare_only_covered_turns(
|
||||
self, tmp_path: Path, transcript: Path, config_dir: Path, estimated_turns: int
|
||||
) -> None:
|
||||
session: Final = statusline_script._session_from_payload(
|
||||
{
|
||||
**RECORDED._asdict(),
|
||||
"spend": 10.0,
|
||||
"baseline_spend": None,
|
||||
"savings_estimated_baseline_spend": 1.5 if estimated_turns else None,
|
||||
"turns": 3,
|
||||
"savings_estimated_turns": estimated_turns,
|
||||
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
|
||||
}
|
||||
)
|
||||
assert session is not None
|
||||
|
||||
def fetch(credentials: Credentials, session_id: str) -> Fetched:
|
||||
return Fetched(session, True)
|
||||
|
||||
first: Final = _run(_payload(transcript), _env(tmp_path, config_dir), fetch)
|
||||
assert first == _run(_payload(transcript), _env(tmp_path, config_dir), fetch)
|
||||
assert first.startswith("Routed to: claude-sonnet-5")
|
||||
if estimated_turns:
|
||||
assert "+33% vs Claude Opus 5 · 1 of 3 turns estimated" in first
|
||||
assert "$2.00" in first and "$1.50" in first
|
||||
assert "$10.00" not in first and "+567%" not in first
|
||||
else:
|
||||
assert "Savings unavailable" in first
|
||||
assert "%" not in first and "$" not in first
|
||||
|
||||
@pytest.mark.parametrize("transcript_model", ("claude-auto", "anthropic/claude-opus-5"))
|
||||
def test_the_session_names_the_routed_model_even_when_the_transcript_differs(
|
||||
self, tmp_path: Path, config_dir: Path, transcript_model: str
|
||||
|
|
|
|||
|
|
@ -281,6 +281,9 @@ class TestFlush:
|
|||
0,
|
||||
"medium",
|
||||
"anthropic/claude-opus-5",
|
||||
0,
|
||||
0.0,
|
||||
0.0,
|
||||
)
|
||||
|
||||
def test_a_connect_error_retries_the_same_statement(self):
|
||||
|
|
@ -307,7 +310,20 @@ class TestFlush:
|
|||
class TestEnqueueSeam:
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("classifier_cost", [0.005, 0.0, None])
|
||||
async def test_update_database_seam_enqueues_only_auto_routed_success(self, classifier_cost: float | None):
|
||||
@pytest.mark.parametrize("estimate, covered, saved", [
|
||||
({"version": 1, "status": "estimated"}, 1, -0.003),
|
||||
({"version": 1, "status": "estimated"}, 1, 0.0),
|
||||
({"version": 2, "status": "estimated"}, 1, 0.0),
|
||||
({"version": 3, "status": "estimated"}, 1, -0.003),
|
||||
({"version": 1, "status": "unknown"}, 0, 0.0),
|
||||
({"version": 0, "status": "estimated"}, 0, 0.0),
|
||||
({"version": 4, "status": "estimated"}, 0, 0.0),
|
||||
({"version": True, "status": "estimated"}, 0, 0.0),
|
||||
(None, 0, -0.003),
|
||||
])
|
||||
async def test_update_database_seam_enqueues_only_auto_routed_success(
|
||||
self, classifier_cost: float | None, estimate: dict[str, object] | None, covered: int, saved: float,
|
||||
) -> None:
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
|
||||
writer: Final = DBSpendUpdateWriter()
|
||||
|
|
@ -315,7 +331,8 @@ class TestEnqueueSeam:
|
|||
_autorouter_turn_transactions_lock=asyncio.Lock(), autorouter_turn_transactions=[]
|
||||
)
|
||||
metadata: Final = _metadata(
|
||||
routing_decision={**ROUTING_DECISION, "classifier_cost": classifier_cost}, autorouter_savings=-0.003
|
||||
routing_decision={**ROUTING_DECISION, "classifier_cost": classifier_cost},
|
||||
autorouter_savings=saved if covered else -0.003, autorouter_savings_estimate=estimate,
|
||||
)
|
||||
for payload in (
|
||||
_payload(metadata=json.dumps(metadata)),
|
||||
|
|
@ -330,7 +347,10 @@ class TestEnqueueSeam:
|
|||
assert transaction.router_name == "live-auto"
|
||||
assert transaction.spend == pytest.approx(0.01 + (classifier_cost or 0.0))
|
||||
assert transaction.classifier_cost == (classifier_cost or 0.0)
|
||||
assert transaction.saved_spend == -0.003
|
||||
assert transaction.saved_spend == saved
|
||||
assert transaction.savings_estimated_turns == covered
|
||||
assert transaction.savings_estimated_actual_spend == pytest.approx(transaction.spend if covered else 0.0)
|
||||
assert transaction.savings_estimated_saved_spend == (saved if covered else 0.0)
|
||||
|
||||
|
||||
def test_every_drain_trigger_reads_the_one_queue_census_owner():
|
||||
|
|
|
|||
|
|
@ -2758,7 +2758,18 @@ async def test_daily_transaction_carries_compression_saved_tokens():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
|
||||
@pytest.mark.parametrize("estimate, recorded_savings, expected", [
|
||||
pytest.param(None, None, -0.005, id="plain-classifier-cost"),
|
||||
pytest.param({"version": 1, "status": "unknown"}, None, 0.0, id="unknown"),
|
||||
pytest.param({"version": 2, "status": "unknown"}, None, 0.0, id="unknown-v2"),
|
||||
pytest.param({"version": 1, "status": "unknown"}, -0.003, 0.0, id="unknown-stale-value"),
|
||||
pytest.param({"version": 0, "status": "estimated"}, -0.003, 0.0, id="unsupported-version"),
|
||||
pytest.param({"version": 1, "status": "estimated"}, -0.003, -0.003, id="estimated"),
|
||||
pytest.param(None, -0.003, -0.003, id="legacy"),
|
||||
])
|
||||
async def test_daily_transaction_compression_saved_tokens_zero_when_absent(
|
||||
estimate: dict[str, object] | None, recorded_savings: float | None, expected: float,
|
||||
) -> None:
|
||||
"""Requests without any compression metadata produce a zero count."""
|
||||
writer = DBSpendUpdateWriter()
|
||||
mock_prisma = MagicMock()
|
||||
|
|
@ -2776,7 +2787,12 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
|
|||
"prompt_tokens": 100,
|
||||
"completion_tokens": 10,
|
||||
"spend": 0.01,
|
||||
"metadata": json.dumps({"usage_object": {}}),
|
||||
"metadata": json.dumps({
|
||||
"usage_object": {"prompt_tokens": 100, "completion_tokens": 10},
|
||||
"routing_decision": {"savings_baseline_model": "anthropic/claude-sonnet-5", "classifier_cost": 0.005},
|
||||
"autorouter_savings": recorded_savings,
|
||||
"autorouter_savings_estimate": estimate,
|
||||
}),
|
||||
}
|
||||
|
||||
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
|
||||
|
|
@ -2789,6 +2805,8 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
|
|||
assert transaction["compression_saved_tokens"] == 0
|
||||
assert transaction["compression_savings_spend"] == 0
|
||||
assert transaction["prompt_caching_savings_spend"] == 0
|
||||
assert transaction["spend"] == 0.01
|
||||
assert transaction["autorouter_savings_spend"] == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
326
tests/test_litellm/proxy/hooks/test_autorouter_baseline_cache.py
Normal file
326
tests/test_litellm/proxy/hooks/test_autorouter_baseline_cache.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Callable, Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Final, cast
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.anthropic.prompt_cache_prediction import NativePredictionTarget, TokenCounter
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import AutoRouterBaselineCache, CapturedBaselineObservation
|
||||
from litellm.router import Router
|
||||
from litellm.types.router import RetryPolicy
|
||||
from litellm.types.utils import CallTypes, StandardLoggingRoutingDecision
|
||||
|
||||
pytestmark: Final = pytest.mark.asyncio
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
_OBJECTS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
|
||||
|
||||
_MESSAGES_JSON: Final = """[{"role":"user","content":[
|
||||
{"type":"text","text":"stable","cache_control":{"type":"ephemeral","ttl":"1h"}},
|
||||
{"type":"text","text":"question"}]}]"""
|
||||
|
||||
|
||||
_MODELS: Final = _MESSAGES.validate_json("""[
|
||||
{"model_name":"test-router","litellm_params":{"model":"auto_router/complexity_router",
|
||||
"complexity_router_config":{"tiers":{"SIMPLE":"sonnet","MEDIUM":"sonnet","COMPLEX":"sonnet",
|
||||
"REASONING":"opus"},"session_affinity":false,
|
||||
"keyword_tier_rules":[{"keywords":["USE_OPUS"],"tier":"REASONING"}]}}},
|
||||
{"model_name":"sonnet","litellm_params":{"model":"anthropic/claude-sonnet-5","api_key":"test-selected"},
|
||||
"model_info":{"id":"selected"}},
|
||||
{"model_name":"opus","litellm_params":{"model":"anthropic/claude-opus-5","api_key":"test-selected"},
|
||||
"model_info":{"id":"baseline"}}]""")
|
||||
|
||||
|
||||
def _message(completed: bool, model: str) -> Mapping[str, JsonValue]:
|
||||
return _JSON_OBJECT.validate_json(f"""{{
|
||||
"id":"msg_baseline_test","type":"message","role":"assistant","model":{json.dumps(model)},
|
||||
"content":{'[{"type":"text","text":"OK"}]' if completed else "[]"},
|
||||
"stop_reason":{'"end_turn"' if completed else "null"},"stop_sequence":null,
|
||||
"usage":{{"input_tokens":1000,"output_tokens":{10 if completed else 0},
|
||||
"cache_creation_input_tokens":5000,"cache_read_input_tokens":0,
|
||||
"cache_creation":{{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5000}}}}}}""")
|
||||
|
||||
|
||||
_EVENTS: Final = _MESSAGES.validate_json("""[
|
||||
{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}},
|
||||
{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"OK"}},
|
||||
{"type":"content_block_stop","index":0},
|
||||
{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}},
|
||||
{"type":"message_stop"}
|
||||
]""")
|
||||
|
||||
|
||||
async def _count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
|
||||
assert model == "claude-opus-5"
|
||||
return 6000 if "question" in json.dumps(_JSON_OBJECT.validate_python(body)) else 5000
|
||||
|
||||
|
||||
class _CallContext(TypedDict):
|
||||
litellm_logging_obj: NotRequired[ReadOnly[Logging]]
|
||||
litellm_call_id: ReadOnly[str]
|
||||
litellm_metadata: ReadOnly[Mapping[str, object]]
|
||||
litellm_session_id: ReadOnly[str]
|
||||
|
||||
|
||||
def _kwargs(logging_obj: Logging, trusted: bool = True, *, explicit_logging: bool = True) -> _CallContext:
|
||||
context: Final = _OBJECTS.validate_json('{"litellm_metadata":{"user_api_key_hash":"test-caller-hash"}}')
|
||||
Router._record_routing_decision( # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # production trusted stamp owner
|
||||
context,
|
||||
StandardLoggingRoutingDecision(
|
||||
router_model_name="test-router",
|
||||
router_type="complexity",
|
||||
routed_model="sonnet",
|
||||
cause="heuristic_scorer",
|
||||
conversation_continuing=True,
|
||||
savings_baseline_model="anthropic/claude-opus-5",
|
||||
savings_baseline_deployment_id="baseline",
|
||||
),
|
||||
)
|
||||
metadata: Final = _OBJECTS.validate_python(context["litellm_metadata"])
|
||||
if not trusted:
|
||||
metadata["_autorouter_baseline_route"] = _JSON_OBJECT.validate_json(
|
||||
'{"router_name":"test-router","baseline_model":"anthropic/claude-opus-5","baseline_deployment_id":"baseline"}'
|
||||
)
|
||||
envelope: Final[_CallContext] = {
|
||||
"litellm_call_id": logging_obj.litellm_call_id,
|
||||
"litellm_session_id": "baseline-session",
|
||||
"litellm_metadata": metadata,
|
||||
}
|
||||
supplied: Final[_CallContext] = {**envelope, "litellm_logging_obj": logging_obj}
|
||||
return supplied if explicit_logging else envelope
|
||||
|
||||
|
||||
def _stream(logging_obj: Logging) -> bool:
|
||||
return logging_obj.stream is True # pyright: ignore[reportUnknownMemberType] # normalize the legacy Logging flag
|
||||
|
||||
|
||||
def _sse(completed: bool = True, model: str = "claude-sonnet-5") -> tuple[bytes, ...]:
|
||||
events: Final = (
|
||||
{ # mutable-ok: json.dumps needs a concrete event dictionary
|
||||
"type": "message_start",
|
||||
"message": _message(False, model),
|
||||
},
|
||||
*_EVENTS,
|
||||
)
|
||||
return tuple(
|
||||
f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
|
||||
for event in (events if completed else events[:-1])
|
||||
)
|
||||
|
||||
|
||||
def _upstream(request: httpx.Request) -> httpx.Response:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.content)
|
||||
model: Final = body.get("model")
|
||||
assert isinstance(model, str)
|
||||
stream: Final = body.get("stream") is True
|
||||
content: Final = b"".join(_sse(model=model)) if stream else json.dumps(_message(True, model)).encode()
|
||||
return httpx.Response(200, content=content, request=request,
|
||||
headers=MappingProxyType({"content-type": "text/event-stream" if stream else "application/json"}),
|
||||
)
|
||||
|
||||
|
||||
def _error(request: httpx.Request, code: int, message: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
code,
|
||||
text='{"type":"error","error":{"type":"rate_limit_error","message":' + json.dumps(message) + "}}",
|
||||
headers=MappingProxyType({"retry-after": "0"}),
|
||||
request=request,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transport(upstream: Callable[[httpx.Request], httpx.Response]) -> Generator[respx.Route]:
|
||||
with respx.mock() as transport:
|
||||
yield transport.post("https://api.anthropic.com/v1/messages").mock(side_effect=upstream)
|
||||
|
||||
|
||||
class _NativeOptions(TypedDict):
|
||||
api_key: NotRequired[ReadOnly[str]]
|
||||
num_retries: NotRequired[ReadOnly[int]]
|
||||
|
||||
|
||||
async def _call(
|
||||
target: Router | None,
|
||||
logging_obj: Logging,
|
||||
*,
|
||||
trusted: bool = True,
|
||||
messages: str = _MESSAGES_JSON,
|
||||
explicit_logging: bool = True,
|
||||
) -> None:
|
||||
invoke: Final = target.anthropic_messages if target else litellm.anthropic_messages # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy native call signatures
|
||||
options: Final = _NativeOptions() if target else _NativeOptions(api_key="test-selected", num_retries=0)
|
||||
response: Final[object] = await invoke( # pyright: ignore[reportUnknownVariableType] # native Router returns an opaque SDK result
|
||||
model="test-router" if target else "anthropic/claude-sonnet-5",
|
||||
max_tokens=16,
|
||||
stream=_stream(logging_obj),
|
||||
messages=_MESSAGES.validate_json(messages),
|
||||
**options,
|
||||
**_kwargs(logging_obj, trusted, explicit_logging=explicit_logging),
|
||||
)
|
||||
assert response is not None
|
||||
if _stream(logging_obj):
|
||||
assert isinstance(response, AsyncIterator)
|
||||
stream: Final = cast(AsyncIterator[object], response) # cast-ok: iterator checked; all items satisfy object
|
||||
assert tuple([chunk async for chunk in stream])
|
||||
|
||||
class _Capture(CustomLogger):
|
||||
def __init__(self, call_id: str) -> None:
|
||||
self.call_id: Final = call_id
|
||||
self.payloads: Final[asyncio.Queue[Mapping[str, object]]] = asyncio.Queue()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
payload: Final = _OBJECTS.validate_python(kwargs.get("standard_logging_object"))
|
||||
if payload.get("litellm_call_id") == self.call_id:
|
||||
self.payloads.put_nowait(payload)
|
||||
|
||||
async def payload(self) -> Mapping[str, object]:
|
||||
return await asyncio.wait_for(self.payloads.get(), timeout=20)
|
||||
|
||||
|
||||
class _Rig:
|
||||
def __init__(self, monkeypatch: pytest.MonkeyPatch, *, retries: int = 0, count: TokenCounter = _count) -> None:
|
||||
self.router: Final = Router(model_list=_MODELS, num_retries=retries,
|
||||
retry_policy=RetryPolicy(RateLimitErrorRetries=retries), disable_cooldowns=True)
|
||||
|
||||
def router() -> Router:
|
||||
return self.router
|
||||
|
||||
self.hook: Final = AutoRouterBaselineCache(None, router=router, token_counter=count)
|
||||
self.call_id: Final = uuid4().hex
|
||||
self.capture: Final = _Capture(self.call_id)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setattr(litellm, "callbacks", [self.hook])
|
||||
for name in ("success_callback", "failure_callback", "_async_failure_callback"):
|
||||
monkeypatch.setattr(litellm, name, [])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [self.capture])
|
||||
|
||||
def logging(self, stream: bool = False) -> Logging:
|
||||
return Logging(model="anthropic/claude-sonnet-5", messages=_MESSAGES.validate_json(_MESSAGES_JSON),
|
||||
stream=stream, call_type=CallTypes.anthropic_messages.value, start_time=datetime.now(),
|
||||
litellm_call_id=self.call_id, function_id=self.call_id, kwargs={"litellm_session_id":"baseline-session"})
|
||||
|
||||
|
||||
def _observation(payload: Mapping[str, object]) -> CapturedBaselineObservation:
|
||||
encoded: Final = payload["autorouter_baseline_observation"]
|
||||
assert isinstance(encoded, str)
|
||||
assert "test-selected" not in encoded and "stable" not in encoded and "x-api-key" not in encoded
|
||||
return CapturedBaselineObservation.model_validate_json(encoded)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream,baseline", ((False, False), (True, False), (False, True), (True, True)))
|
||||
async def test_native_logging_captures_usage_without_publishing_hypothetical_savings(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool, baseline: bool,
|
||||
) -> None:
|
||||
rig: Final = _Rig(monkeypatch)
|
||||
messages: Final = _MESSAGES_JSON.replace("question", "question USE_OPUS") if baseline else _MESSAGES_JSON
|
||||
with _transport(_upstream):
|
||||
await _call(rig.router, rig.logging(stream), messages=messages)
|
||||
payload: Final = await rig.capture.payload()
|
||||
captured: Final = _observation(payload)
|
||||
assert payload["autorouter_savings"] is None
|
||||
assert _OBJECTS.validate_python(payload["autorouter_savings_estimate"])["reason"] == "pending_projection"
|
||||
assert captured.observation.outcome == "complete"
|
||||
assert captured.observation.baseline_equivalent == baseline
|
||||
assert captured.observation.usage is not None and captured.observation.usage.completion_tokens == 10
|
||||
assert captured.observation.plan is not None and captured.observation.plan.total_tokens == 6000
|
||||
|
||||
|
||||
async def test_count_failure_preserves_initial_observed_equivalence(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
|
||||
return None
|
||||
|
||||
rig: Final = _Rig(monkeypatch, count=count)
|
||||
with _transport(_upstream):
|
||||
await _call(rig.router, rig.logging(), messages=_MESSAGES_JSON.replace("question", "question USE_OPUS"))
|
||||
captured: Final = _observation(await rig.capture.payload())
|
||||
assert captured.observation.baseline_equivalent and captured.observation.usage is not None
|
||||
assert captured.observation.plan is None and captured.observation.reason == "token_count_unavailable"
|
||||
|
||||
|
||||
async def test_native_retry_is_uncertain_even_when_final_response_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
rig: Final = _Rig(monkeypatch, retries=1)
|
||||
|
||||
def upstream(request: httpx.Request) -> httpx.Response:
|
||||
return _upstream(request) if route.call_count else _error(request, 429, "retry")
|
||||
|
||||
with _transport(upstream) as route:
|
||||
await _call(rig.router, rig.logging())
|
||||
captured: Final = _observation(await rig.capture.payload())
|
||||
assert route.call_count == 2
|
||||
assert captured.observation.outcome == "uncertain"
|
||||
assert captured.observation.reason == "retried_request"
|
||||
|
||||
|
||||
async def test_caller_cannot_forge_an_observation_scope(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
rig: Final = _Rig(monkeypatch)
|
||||
with _transport(_upstream):
|
||||
await _call(None, rig.logging(), trusted=False)
|
||||
payload: Final = await rig.capture.payload()
|
||||
assert payload["autorouter_baseline_observation"] is None
|
||||
assert payload["autorouter_savings"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,key,endpoint", (
|
||||
("claude-sonnet-5", "test-first", None),
|
||||
("claude-opus-5", "test-second", None),
|
||||
("claude-opus-5", "test-first", "https://example.test"),
|
||||
))
|
||||
async def test_count_memo_is_scoped_to_provider_recipient(model: str, key: str, endpoint: str | None) -> None:
|
||||
counts: Final = iter((5000, 6000))
|
||||
|
||||
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
|
||||
return next(counts)
|
||||
|
||||
collector: Final = AutoRouterBaselineCache(None, token_counter=count)
|
||||
original: Final = NativePredictionTarget("claude-opus-5", "test-first")
|
||||
other: Final = NativePredictionTarget(model, key, endpoint)
|
||||
assert await collector._count(original, {}) == 5000 # pyright: ignore[reportPrivateUsage]
|
||||
assert await collector._count(other, {}) == 6000 # pyright: ignore[reportPrivateUsage]
|
||||
assert await collector._count(original, {}) == 5000 # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
async def test_provider_counting_does_not_hold_the_inference_response(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool,
|
||||
) -> None:
|
||||
counting: Final = asyncio.Event()
|
||||
release: Final = asyncio.Event()
|
||||
|
||||
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
|
||||
counting.set()
|
||||
await release.wait()
|
||||
return await _count(model, api_key, body)
|
||||
|
||||
rig: Final = _Rig(monkeypatch, count=count)
|
||||
try:
|
||||
with _transport(_upstream):
|
||||
await asyncio.wait_for(_call(rig.router, rig.logging(stream)), timeout=2)
|
||||
await asyncio.wait_for(counting.wait(), timeout=2)
|
||||
assert rig.capture.payloads.empty()
|
||||
release.set()
|
||||
assert _observation(await rig.capture.payload()).observation.plan is not None
|
||||
finally:
|
||||
release.set()
|
||||
|
|
@ -546,6 +546,9 @@ class TestAutoRouterBenchmarks:
|
|||
total_tokens=4000,
|
||||
spend=10.0,
|
||||
saved_spend=30.0,
|
||||
savings_estimated_turns=40,
|
||||
savings_estimated_actual_spend=10.0,
|
||||
savings_estimated_saved_spend=30.0,
|
||||
classifier_cost=0.4,
|
||||
classifier_cost_recorded_turns=40,
|
||||
session_seconds=400.0,
|
||||
|
|
@ -582,12 +585,29 @@ class TestAutoRouterBenchmarks:
|
|||
def test_a_losing_router_reports_negative_savings(self):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
|
||||
|
||||
losing = self.ROW.model_copy(update={"saved_spend": -5.0})
|
||||
losing = self.ROW.model_copy(update={"saved_spend": -5.0, "savings_estimated_saved_spend": -5.0})
|
||||
totals = _benchmark_totals(losing)
|
||||
assert totals.baseline_spend == 5.0
|
||||
assert totals.saved_pct == -100.0
|
||||
assert totals.classifier_cost == 0.4
|
||||
|
||||
@pytest.mark.parametrize("estimated_turns", [0, 4])
|
||||
def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None:
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
|
||||
|
||||
row: Final = self.ROW.model_copy(update={
|
||||
"savings_estimated_turns": estimated_turns,
|
||||
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
|
||||
"savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0,
|
||||
})
|
||||
totals: Final = _benchmark_totals(row)
|
||||
assert totals.spend == 10.0
|
||||
assert totals.savings_estimated_turns == estimated_turns
|
||||
assert totals.saved_spend == (-0.5 if estimated_turns else None)
|
||||
assert totals.baseline_spend == (1.5 if estimated_turns else None)
|
||||
assert totals.saved_pct == (pytest.approx(-33.3) if estimated_turns else None)
|
||||
assert totals.saved_per_session is None
|
||||
|
||||
def test_an_empty_window_folds_to_zeros(self):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import (
|
||||
_benchmark_totals,
|
||||
|
|
@ -607,7 +627,10 @@ class TestAutoRouterBenchmarks:
|
|||
_summed_agg_row,
|
||||
)
|
||||
|
||||
other = self.ROW.model_copy(update={"router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0})
|
||||
other = self.ROW.model_copy(update={
|
||||
"router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0,
|
||||
"savings_estimated_turns": 10, "savings_estimated_actual_spend": 0.0,
|
||||
})
|
||||
summed = _summed_agg_row([self.ROW, other])
|
||||
totals = _benchmark_totals(summed)
|
||||
assert summed.sessions == 5
|
||||
|
|
@ -696,6 +719,9 @@ class TestAutoRouterBenchmarks:
|
|||
"turns": 10,
|
||||
"spend": 2.0,
|
||||
"saved_spend": -0.5,
|
||||
"savings_estimated_turns": 10,
|
||||
"savings_estimated_actual_spend": 2.0,
|
||||
"savings_estimated_saved_spend": -0.5,
|
||||
"classifier_cost": recorded_turns * 0.02,
|
||||
"classifier_cost_recorded_turns": recorded_turns,
|
||||
}
|
||||
|
|
@ -876,6 +902,10 @@ class TestAutoRouterSession:
|
|||
"last_model": "anthropic/claude-sonnet-5",
|
||||
"spend": 0.14,
|
||||
"saved_spend": 0.24,
|
||||
"savings_estimated_turns": 3,
|
||||
"savings_estimated_actual_spend": 0.14,
|
||||
"savings_estimated_saved_spend": 0.24,
|
||||
"savings_estimated_baseline_models": {"anthropic/claude-opus-5": 3},
|
||||
"classifier_cost": 0.0,
|
||||
"tier_turns": {"simple": 1, "complex": 2},
|
||||
"baseline_models": {"anthropic/claude-opus-5": 3},
|
||||
|
|
@ -899,25 +929,33 @@ class TestAutoRouterSession:
|
|||
return lookups
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"])
|
||||
async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
self, monkeypatch: pytest.MonkeyPatch, turns: int, estimated: bool,
|
||||
) -> None:
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
|
||||
|
||||
caller = UserAPIKeyAuth(api_key="sk-caller")
|
||||
self._rig(monkeypatch, [{**self.ROW, "api_key": caller.api_key, "session_id": "sess-1"}])
|
||||
row: Final = {key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")}
|
||||
spend: Final = 0.14 if turns == 3 else 10.0
|
||||
if estimated and turns != 3:
|
||||
row["savings_estimated_saved_spend"] = -0.04
|
||||
self._rig(monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}])
|
||||
response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1")
|
||||
assert response.model_dump() == {
|
||||
"session_id": "sess-1",
|
||||
"router_name": "claude-auto",
|
||||
"router_type": "complexity",
|
||||
"turns": 3,
|
||||
"turns": turns,
|
||||
"last_model": "anthropic/claude-sonnet-5",
|
||||
"spend": 0.14,
|
||||
"saved_spend": 0.24,
|
||||
"baseline_spend": pytest.approx(0.38),
|
||||
"baseline_model": "anthropic/claude-opus-5",
|
||||
"baseline_models": {"anthropic/claude-opus-5": 3},
|
||||
"spend": spend,
|
||||
"saved_spend": (0.24 if turns == 3 else -0.04) if estimated else None,
|
||||
"savings_estimated_turns": 3 if estimated else 0,
|
||||
"savings_estimated_actual_spend": 0.14 if estimated else 0.0,
|
||||
"baseline_spend": pytest.approx(0.38) if turns == 3 else None,
|
||||
"savings_estimated_baseline_spend": pytest.approx(0.38 if turns == 3 else 0.1) if estimated else None,
|
||||
"baseline_model": "anthropic/claude-opus-5" if estimated else None,
|
||||
"baseline_models": {"anthropic/claude-opus-5": 3} if estimated else {},
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -959,22 +997,14 @@ class TestAutoRouterSession:
|
|||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
|
||||
|
||||
priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1}
|
||||
self._rig(monkeypatch, [{**self.ROW, "api_key": ADMIN.api_key, "session_id": "s", "baseline_models": priced}])
|
||||
self._rig(monkeypatch, [{
|
||||
**self.ROW, "api_key": ADMIN.api_key, "session_id": "s",
|
||||
"baseline_models": {"old-baseline": 100}, "savings_estimated_baseline_models": priced,
|
||||
}])
|
||||
response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
|
||||
assert response.baseline_model == "anthropic/claude-opus-5"
|
||||
assert response.baseline_models == priced
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_session_whose_turns_recorded_no_baseline_reports_the_money_without_a_name(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
|
||||
|
||||
self._rig(monkeypatch, [{**self.ROW, "api_key": ADMIN.api_key, "session_id": "s", "baseline_models": {}}])
|
||||
response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
|
||||
assert response.baseline_model is None
|
||||
assert response.baseline_spend == pytest.approx(0.38)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_oversized_client_session_id_is_bounded_like_the_writer_bounded_it(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
|
||||
import pytest
|
||||
|
||||
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
apply_unified_file_ids,
|
||||
get_credentials_for_model,
|
||||
is_litellm_executed_batch,
|
||||
map_raw_file_ids_to_unified,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
|
@ -500,3 +500,17 @@ class TestCompletedBatchSafeToRetire:
|
|||
|
||||
def test_no_output_and_unknown_counts_is_not_safe(self):
|
||||
assert _completed_batch_safe_to_retire(_completed_batch_for_retire(None)) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"decoded_unified_batch_id, executed",
|
||||
[
|
||||
("litellm_proxy;model_id:my-vllm;llm_batch_id:litellm_batch_0123abcd", True),
|
||||
("litellm_proxy;model_id:my-vllm;llm_batch_id:batch_0123abcd", False),
|
||||
("litellm_proxy;model_id:my-vllm;generic_response_id:resp_0123abcd", False),
|
||||
("litellm_proxy;model_id:my-vllm;llm_output_file_id:file-0123abcd", False),
|
||||
("batch_0123abcd", False),
|
||||
],
|
||||
)
|
||||
def test_is_litellm_executed_batch_reads_the_llm_batch_id_prefix(decoded_unified_batch_id: str, executed: bool):
|
||||
assert is_litellm_executed_batch(decoded_unified_batch_id) is executed
|
||||
|
|
|
|||
|
|
@ -609,6 +609,246 @@ def test_target_storage_with_target_models(
|
|||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
BATCH_JSONL_LINE = (
|
||||
b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", '
|
||||
b'"body": {"model": "my-vllm", "messages": [{"role": "user", "content": "hi"}]}}\n'
|
||||
)
|
||||
|
||||
|
||||
def _router_with_executed_batch_model() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-vllm",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/qwen",
|
||||
"api_key": "sk-vllm",
|
||||
"api_base": "http://vllm.test/v1",
|
||||
},
|
||||
"model_info": {"id": "my-vllm-id"},
|
||||
},
|
||||
{
|
||||
"model_name": "gemini-2.0-flash",
|
||||
"litellm_params": {"model": "gemini/gemini-2.0-flash"},
|
||||
"model_info": {"id": "gemini-2.0-flash-id"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def batch_upload_seams(mocker: MockerFixture, monkeypatch):
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
llm_router = _router_with_executed_batch_model()
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router)
|
||||
setup_proxy_logging_object(monkeypatch, llm_router)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test-user"
|
||||
)
|
||||
uploaded = OpenAIFileObject(
|
||||
id="file-kept",
|
||||
object="file",
|
||||
purpose="batch",
|
||||
created_at=0,
|
||||
bytes=len(BATCH_JSONL_LINE),
|
||||
filename="batch.jsonl",
|
||||
status="uploaded",
|
||||
)
|
||||
stored = mocker.patch( # test-quality-ok: the route calls the storage service directly with no injection seam
|
||||
"litellm.proxy.openai_files_endpoints.storage_backend_service.StorageBackendFileService.upload_file_to_storage_backend",
|
||||
new=mocker.AsyncMock(return_value=uploaded),
|
||||
)
|
||||
provider_upload = mocker.patch( # test-quality-ok: the route calls litellm.acreate_file directly with no injection seam
|
||||
"litellm.acreate_file", new=mocker.AsyncMock(return_value=uploaded)
|
||||
)
|
||||
try:
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream_files_route = upstream.get("http://vllm.test/v1/files").mock(
|
||||
return_value=httpx.Response(404, json={"detail": "Not Found"})
|
||||
)
|
||||
yield stored, provider_upload, upstream_files_route
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
def _upload_batch_file(headers: dict[str, str], form: dict[str, str]):
|
||||
return client.post(
|
||||
"/v1/files",
|
||||
files={"file": ("batch.jsonl", BATCH_JSONL_LINE, "application/jsonl")},
|
||||
data={"purpose": "batch", **form},
|
||||
headers={"Authorization": "Bearer test-key", **headers},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"headers, form",
|
||||
[({"x-litellm-model": "my-vllm"}, {}), ({}, {"target_model_names": "my-vllm"})],
|
||||
ids=["x-litellm-model header", "target_model_names form field"],
|
||||
)
|
||||
def test_batch_upload_for_a_litellm_executed_model_is_kept_by_litellm(
|
||||
batch_upload_seams, headers: dict[str, str], form: dict[str, str]
|
||||
):
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file(headers, form)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
provider_upload.assert_not_awaited()
|
||||
stored.assert_awaited_once()
|
||||
kwargs = stored.call_args.kwargs
|
||||
assert kwargs["target_storage"] == "litellm_db"
|
||||
assert tuple(kwargs["target_model_names"]) == ("my-vllm",)
|
||||
assert kwargs["purpose"] == "batch"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"headers, form",
|
||||
[({"x-litellm-model": "my-vllm"}, {}), ({}, {"target_model_names": "my-vllm"})],
|
||||
ids=["x-litellm-model header", "target_model_names form field"],
|
||||
)
|
||||
def test_batch_upload_for_a_litellm_executed_model_the_key_cannot_call_is_refused_before_the_server_is_probed(
|
||||
batch_upload_seams, headers: dict[str, str], form: dict[str, str]
|
||||
):
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
||||
stored, provider_upload, upstream_files_route = batch_upload_seams
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="restricted-user", models=["gemini-2.0-flash"]
|
||||
)
|
||||
|
||||
response = _upload_batch_file(headers, form)
|
||||
|
||||
assert response.status_code == 403, response.text
|
||||
assert "my-vllm" in response.text
|
||||
assert upstream_files_route.call_count == 0
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
def test_batch_upload_naming_an_executed_and_a_provider_model_is_rejected(batch_upload_seams):
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({}, {"target_model_names": "my-vllm,gemini-2.0-flash"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert "my-vllm" in response.text
|
||||
assert "target_model_names" in response.text
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ["assistants", "user_data"])
|
||||
def test_non_batch_upload_for_a_litellm_executed_model_is_rejected_with_the_purpose_to_use(
|
||||
batch_upload_seams, purpose: str
|
||||
):
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({"x-litellm-model": "my-vllm"}, {"purpose": purpose})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
assert error["param"] == "purpose"
|
||||
assert "purpose=batch" in error["message"]
|
||||
assert f"purpose={purpose}" in error["message"]
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ["batch", "assistants"])
|
||||
@pytest.mark.parametrize(
|
||||
"upstream_answer",
|
||||
[httpx.Response(200, json={"object": "list", "data": []}), httpx.Response(405), httpx.ConnectError("refused")],
|
||||
ids=["lists files", "files route without list", "unreachable"],
|
||||
)
|
||||
def test_upload_for_a_litellm_executed_model_goes_to_the_provider_unless_the_server_lacks_a_files_api(
|
||||
batch_upload_seams, upstream_answer: httpx.Response | httpx.ConnectError, purpose: str
|
||||
):
|
||||
stored, provider_upload, upstream_files_route = batch_upload_seams
|
||||
upstream_files_route.mock(side_effect=[upstream_answer])
|
||||
|
||||
response = _upload_batch_file({"x-litellm-model": "my-vllm"}, {"purpose": purpose})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_awaited_once()
|
||||
assert provider_upload.call_args.kwargs["custom_llm_provider"] == "hosted_vllm"
|
||||
assert provider_upload.call_args.kwargs["api_base"] == "http://vllm.test/v1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"form",
|
||||
[{}, {"target_model_names": "my-vllm"}, {"target_model_names": "gemini-2.0-flash"}],
|
||||
ids=["no model", "litellm-executed model", "provider model"],
|
||||
)
|
||||
def test_upload_naming_litellm_db_as_target_storage_is_rejected(batch_upload_seams, form: dict[str, str]):
|
||||
stored, provider_upload, upstream_files_route = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({}, {**form, "target_storage": "litellm_db"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
assert error["param"] == "target_storage"
|
||||
assert "litellm_db" in error["message"]
|
||||
assert upstream_files_route.call_count == 0
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ["user_data", "batch"])
|
||||
def test_upload_with_an_explicit_target_storage_goes_where_the_caller_said_without_probing_the_server(
|
||||
batch_upload_seams, purpose: str
|
||||
):
|
||||
stored, provider_upload, upstream_files_route = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file(
|
||||
{}, {"purpose": purpose, "target_model_names": "my-vllm", "target_storage": "azure_storage"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert upstream_files_route.call_count == 0
|
||||
provider_upload.assert_not_awaited()
|
||||
stored.assert_awaited_once()
|
||||
kwargs = stored.call_args.kwargs
|
||||
assert kwargs["target_storage"] == "azure_storage"
|
||||
assert tuple(kwargs["target_model_names"]) == ("my-vllm",)
|
||||
assert kwargs["purpose"] == purpose
|
||||
|
||||
|
||||
def test_upload_with_an_explicit_target_storage_still_refuses_a_key_without_the_executed_model(batch_upload_seams):
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
||||
stored, provider_upload, upstream_files_route = batch_upload_seams
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="restricted-user", models=["gemini-2.0-flash"]
|
||||
)
|
||||
|
||||
response = _upload_batch_file({}, {"target_model_names": "my-vllm", "target_storage": "azure_storage"})
|
||||
|
||||
assert response.status_code == 403, response.text
|
||||
assert "my-vllm" in response.text
|
||||
assert upstream_files_route.call_count == 0
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
def test_batch_upload_for_a_provider_model_still_goes_to_the_provider(batch_upload_seams):
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({"x-litellm-model": "gemini-2.0-flash"}, {})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_awaited_once()
|
||||
assert provider_upload.call_args.kwargs["custom_llm_provider"] == "gemini"
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="mock respx fails on ci/cd - unclear why")
|
||||
def test_create_file_and_call_chat_completion_e2e(
|
||||
mocker: MockerFixture, monkeypatch, llm_router: Router
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
|
||||
|
|
@ -6,16 +8,24 @@ from litellm.proxy.openai_files_endpoints import storage_backend_service
|
|||
from litellm.proxy.openai_files_endpoints.storage_backend_service import (
|
||||
StorageBackendFileService,
|
||||
)
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
class _RecordingStorageBackend:
|
||||
def __init__(self):
|
||||
def __init__(self, delete_error: Exception | None = None):
|
||||
self.upload_calls = []
|
||||
self.delete_calls: list[str] = []
|
||||
self.delete_error = delete_error
|
||||
|
||||
async def upload_file(self, **kwargs):
|
||||
self.upload_calls.append(kwargs)
|
||||
return "https://storage.example/blob-1"
|
||||
|
||||
async def delete_file(self, storage_url: str) -> None:
|
||||
self.delete_calls.append(storage_url)
|
||||
if self.delete_error is not None:
|
||||
raise self.delete_error
|
||||
|
||||
|
||||
class _FakeManagedFilesHook(BaseFileEndpoints):
|
||||
def __init__(self):
|
||||
|
|
@ -42,6 +52,11 @@ class _FakeManagedFilesHook(BaseFileEndpoints):
|
|||
self.stored.append(kwargs)
|
||||
|
||||
|
||||
class _FailingManagedFilesHook(_FakeManagedFilesHook):
|
||||
async def store_unified_file_id(self, **kwargs):
|
||||
raise RuntimeError("db down")
|
||||
|
||||
|
||||
class _FakeProxyLogging:
|
||||
def __init__(self, hook):
|
||||
self._hook = hook
|
||||
|
|
@ -57,7 +72,7 @@ def _file_data():
|
|||
@pytest.mark.asyncio
|
||||
async def test_upload_with_target_model_names_but_no_hook_raises_before_uploading(monkeypatch):
|
||||
backend = _RecordingStorageBackend()
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name: backend)
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name, prisma_client=None: backend)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
|
|
@ -80,7 +95,7 @@ async def test_upload_with_target_model_names_but_no_hook_raises_before_uploadin
|
|||
@pytest.mark.asyncio
|
||||
async def test_upload_without_target_model_names_skips_hook_requirement(monkeypatch):
|
||||
backend = _RecordingStorageBackend()
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name: backend)
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name, prisma_client=None: backend)
|
||||
|
||||
file_object = await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
file_data=_file_data(),
|
||||
|
|
@ -101,7 +116,7 @@ async def test_upload_without_target_model_names_skips_hook_requirement(monkeypa
|
|||
@pytest.mark.asyncio
|
||||
async def test_upload_with_target_model_names_and_hook_stores_unified_id(monkeypatch):
|
||||
backend = _RecordingStorageBackend()
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name: backend)
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name, prisma_client=None: backend)
|
||||
hook = _FakeManagedFilesHook()
|
||||
|
||||
file_object = await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
|
|
@ -125,3 +140,50 @@ async def test_upload_with_target_model_names_and_hook_stores_unified_id(monkeyp
|
|||
"stored_id_matches_response": True,
|
||||
"model_mappings": {"gpt-x": "https://storage.example/blob-1"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_hands_the_prisma_client_to_the_storage_backend_factory(monkeypatch: pytest.MonkeyPatch):
|
||||
backend = _RecordingStorageBackend()
|
||||
factory_calls: list[tuple[str, PrismaClient | None]] = []
|
||||
|
||||
def _factory(name: str, prisma_client: PrismaClient | None = None) -> _RecordingStorageBackend:
|
||||
factory_calls.append((name, prisma_client))
|
||||
return backend
|
||||
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", _factory)
|
||||
prisma_client = MagicMock()
|
||||
|
||||
await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
file_data=_file_data(),
|
||||
target_storage="litellm_db",
|
||||
target_model_names=[],
|
||||
purpose="batch",
|
||||
proxy_logging_obj=_FakeProxyLogging(hook=None),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
|
||||
assert factory_calls == [("litellm_db", prisma_client)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("delete_error", [None, OSError("blob locked")], ids=["delete succeeds", "delete fails"])
|
||||
async def test_upload_deletes_the_uploaded_content_when_the_metadata_write_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, delete_error: Exception | None
|
||||
):
|
||||
backend = _RecordingStorageBackend(delete_error=delete_error)
|
||||
monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name, prisma_client=None: backend)
|
||||
|
||||
with pytest.raises(RuntimeError, match="db down"):
|
||||
await StorageBackendFileService.upload_file_to_storage_backend(
|
||||
file_data=_file_data(),
|
||||
target_storage="azure_storage",
|
||||
target_model_names=["gpt-x"],
|
||||
purpose="batch",
|
||||
proxy_logging_obj=_FakeProxyLogging(hook=_FailingManagedFilesHook()),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
)
|
||||
|
||||
assert len(backend.upload_calls) == 1
|
||||
assert backend.delete_calls == ["https://storage.example/blob-1"]
|
||||
|
|
|
|||
|
|
@ -4526,3 +4526,191 @@ async def test_add_deployment_syncs_ui_settings_even_when_the_model_reconcile_fa
|
|||
await config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=MagicMock())
|
||||
|
||||
assert general_settings["allow_agents_for_team_admins"] is True
|
||||
|
||||
|
||||
def _websearch_logger_cls():
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
return WebSearchInterceptionLogger
|
||||
|
||||
|
||||
def _run_websearch_init(monkeypatch, stored_params, starting_callbacks):
|
||||
pc = ProxyConfig()
|
||||
monkeypatch.setattr(litellm, "callbacks", list(starting_callbacks))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.get_config_param",
|
||||
AsyncMock(return_value=SimpleNamespace(param_value={"websearch_interception_params": stored_params}))
|
||||
if stored_params is not None
|
||||
else AsyncMock(return_value=SimpleNamespace(param_value={})),
|
||||
)
|
||||
asyncio.run(pc.init_websearch_interception_settings_in_db(prisma_client=MagicMock()))
|
||||
return pc
|
||||
|
||||
|
||||
def _poll_websearch_init(pc, monkeypatch, stored_params):
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.get_config_param",
|
||||
AsyncMock(return_value=SimpleNamespace(param_value={"websearch_interception_params": stored_params})),
|
||||
)
|
||||
asyncio.run(pc.init_websearch_interception_settings_in_db(prisma_client=MagicMock()))
|
||||
|
||||
|
||||
def test_init_websearch_interception_resyncs_after_a_write_drops_the_enabled_flag(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
pc = ProxyConfig()
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
|
||||
_poll_websearch_init(pc, monkeypatch, {"enabled": True, "search_tool_name": "old-tool"})
|
||||
_poll_websearch_init(pc, monkeypatch, {"search_tool_name": "new-tool"})
|
||||
|
||||
registered = [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)]
|
||||
assert len(registered) == 1
|
||||
assert registered[0].search_tool_name == "new-tool"
|
||||
|
||||
|
||||
def test_init_websearch_interception_ignores_a_non_list_providers_value(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": True, "enabled_providers": "bedrock", "search_tool_name": "stored-tool"},
|
||||
starting_callbacks=[],
|
||||
)
|
||||
|
||||
registered = [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)]
|
||||
assert len(registered) == 1
|
||||
assert registered[0].enabled_providers == ["bedrock"]
|
||||
|
||||
|
||||
def test_init_websearch_interception_absent_key_leaves_callbacks_untouched(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
config_registered = logger_cls(search_tool_name="from-config-yaml")
|
||||
|
||||
_run_websearch_init(monkeypatch, stored_params=None, starting_callbacks=[config_registered])
|
||||
|
||||
assert litellm.callbacks == [config_registered]
|
||||
|
||||
|
||||
def test_init_websearch_interception_without_enabled_key_leaves_callbacks_untouched(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
config_registered = logger_cls(search_tool_name="from-config-yaml")
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"search_tool_name": "stored-tool"},
|
||||
starting_callbacks=[config_registered],
|
||||
)
|
||||
|
||||
assert litellm.callbacks == [config_registered]
|
||||
|
||||
|
||||
def test_init_websearch_interception_registers_when_explicitly_enabled(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": True, "search_tool_name": "stored-tool"},
|
||||
starting_callbacks=[],
|
||||
)
|
||||
|
||||
registered = [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)]
|
||||
assert len(registered) == 1
|
||||
assert registered[0].search_tool_name == "stored-tool"
|
||||
|
||||
|
||||
def test_init_websearch_interception_treats_string_false_as_disabled(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
existing = logger_cls(search_tool_name="stored-tool")
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": "false", "search_tool_name": "stored-tool"},
|
||||
starting_callbacks=[existing],
|
||||
)
|
||||
|
||||
assert [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)] == []
|
||||
|
||||
|
||||
def test_init_websearch_interception_empty_providers_falls_back_to_handler_default(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": True, "enabled_providers": [], "search_tool_name": "stored-tool"},
|
||||
starting_callbacks=[],
|
||||
)
|
||||
|
||||
registered = [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)]
|
||||
assert len(registered) == 1
|
||||
assert registered[0].enabled_providers == ["bedrock"]
|
||||
|
||||
|
||||
def test_init_websearch_interception_keeps_working_callback_when_new_one_cannot_be_built(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
working = logger_cls(search_tool_name="stored-tool", max_agentic_loops=3)
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": True, "search_tool_name": "stored-tool", "max_agentic_loops": 0},
|
||||
starting_callbacks=[working],
|
||||
)
|
||||
|
||||
assert litellm.callbacks == [working]
|
||||
|
||||
|
||||
def test_init_websearch_interception_disabled_removes_the_callback(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
existing = logger_cls(search_tool_name="stored-tool")
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": False, "search_tool_name": "stored-tool"},
|
||||
starting_callbacks=[existing],
|
||||
)
|
||||
|
||||
assert [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)] == []
|
||||
|
||||
|
||||
def test_init_websearch_interception_replaces_stale_instance_on_param_change(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
stale = logger_cls(search_tool_name="old-tool", max_agentic_loops=2)
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": True, "search_tool_name": "new-tool", "max_agentic_loops": 7},
|
||||
starting_callbacks=[stale],
|
||||
)
|
||||
|
||||
registered = [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)]
|
||||
assert len(registered) == 1
|
||||
assert (registered[0].search_tool_name, registered[0].max_agentic_loops) == ("new-tool", 7)
|
||||
|
||||
|
||||
def test_init_websearch_interception_honors_enabled_providers(monkeypatch):
|
||||
logger_cls = _websearch_logger_cls()
|
||||
|
||||
_run_websearch_init(
|
||||
monkeypatch,
|
||||
stored_params={"enabled": True, "enabled_providers": ["bedrock", "vertex_ai"]},
|
||||
starting_callbacks=[],
|
||||
)
|
||||
|
||||
registered = [cb for cb in litellm.callbacks if isinstance(cb, logger_cls)]
|
||||
assert len(registered) == 1
|
||||
assert registered[0].enabled_providers == ["bedrock", "vertex_ai"]
|
||||
|
||||
|
||||
def test_websearch_interception_settings_can_be_named_in_supported_db_objects(monkeypatch):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ConfigGeneralSettings
|
||||
|
||||
allowlist = ConfigGeneralSettings(supported_db_objects=["websearch_interception_settings"]).supported_db_objects
|
||||
assert allowlist
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": allowlist})
|
||||
assert proxy_server.should_load_db_object(object_type="websearch_interception_settings") is True
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]})
|
||||
assert proxy_server.should_load_db_object(object_type="websearch_interception_settings") is False
|
||||
|
|
|
|||
|
|
@ -11,12 +11,23 @@ from fastapi.testclient import TestClient
|
|||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.public_endpoints import router
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelGroupInfoProxy,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
def test_fuse_presets_route_serves_the_shared_catalog_without_authentication() -> None:
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
client: Final = TestClient(app)
|
||||
response: Final = client.get("/public/complexity_router/fuse_presets")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == get_fuse_presets().model_dump(mode="json")
|
||||
assert client.get("/public/complexity_router/fuse_presets").json() == response.json()
|
||||
|
||||
|
||||
def test_get_supported_providers_returns_enum_values():
|
||||
app_instance = FastAPI()
|
||||
app_instance.include_router(router)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,199 @@
|
|||
from dataclasses import replace
|
||||
from itertools import groupby
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.cost_calculation import cost_per_token
|
||||
from litellm.llms.anthropic.prompt_cache_prediction import CountedBreakpoint, CountedPromptCachePlan
|
||||
from litellm.proxy.spend_tracking.baseline_accounting import (
|
||||
BaselineEstimate,
|
||||
BaselineHistory,
|
||||
BaselineObservation,
|
||||
CacheEntry,
|
||||
advance_baseline_history,
|
||||
)
|
||||
from litellm.types.utils import CacheCreationTokenDetails, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
|
||||
def _usage() -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=6200,
|
||||
completion_tokens=30,
|
||||
total_tokens=6230,
|
||||
cache_read_input_tokens=0,
|
||||
cache_creation_input_tokens=6000,
|
||||
speed="fast",
|
||||
inference_geo="us",
|
||||
completion_tokens_details={"reasoning_tokens": 20},
|
||||
server_tool_use={"web_search_requests": 1},
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=200,
|
||||
cached_tokens=0,
|
||||
cache_creation_tokens=6000,
|
||||
cache_write_tokens=6000,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=0, ephemeral_1h_input_tokens=6000
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _marker(
|
||||
name: str = "prefix", ttl: int = 3600, tokens: int = 6000, previous: tuple[str, ...] = ()
|
||||
) -> CountedBreakpoint:
|
||||
return CountedBreakpoint(
|
||||
fingerprint=f"{name}:{ttl}",
|
||||
ttl_seconds=ttl,
|
||||
prefix_tokens=tokens,
|
||||
lookback_fingerprints=(*(f"{item}:{ttl}" for item in previous), f"{name}:{ttl}"),
|
||||
content_fingerprint=name,
|
||||
lookback_content_fingerprints=(*previous, name),
|
||||
)
|
||||
|
||||
|
||||
def _observation(request_id: str, started: float = 10000.0, **overrides: object) -> BaselineObservation:
|
||||
return BaselineObservation.model_validate(
|
||||
{
|
||||
"request_id": request_id,
|
||||
"started_at": started,
|
||||
"available_at": started + 0.1,
|
||||
"outcome": "complete",
|
||||
"baseline_equivalent": False,
|
||||
"usage": _usage(),
|
||||
"plan": CountedPromptCachePlan(6200, (_marker(),)),
|
||||
"minimum_cache_tokens": 4096,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _replay(*observations: BaselineObservation) -> tuple[BaselineEstimate, ...]:
|
||||
history = BaselineHistory()
|
||||
results: list[BaselineEstimate] = []
|
||||
for _, group in groupby(sorted(observations, key=lambda item: item.started_at), key=lambda item: item.started_at):
|
||||
history, estimates = advance_baseline_history(history, tuple(group))
|
||||
results.extend(estimates)
|
||||
return tuple(results)
|
||||
|
||||
|
||||
def test_initial_identical_path_preserves_full_usage_without_counting_or_exclusive_owner() -> None:
|
||||
initial: Final = _observation("main", baseline_equivalent=True, plan=None, reason="unsupported_request_headers")
|
||||
background: Final = initial.model_copy(update={"request_id": "background"})
|
||||
later: Final = initial.model_copy(update={"request_id": "later", "started_at": 10001.0, "available_at": 10002.0})
|
||||
estimates: Final = _replay(initial, background, later)
|
||||
assert all(item.provenance == "observed_identical" and item.usage == initial.usage for item in estimates)
|
||||
assert all(item.usage is not initial.usage for item in estimates)
|
||||
assert all(item.usage.prompt_tokens == 6200 for item in estimates if item.usage is not None)
|
||||
|
||||
|
||||
def test_late_divergent_observation_replays_in_event_order_and_removes_initial_zero() -> None:
|
||||
same: Final = _observation("same", 10001.0, baseline_equivalent=True)
|
||||
early: Final = _observation("early")
|
||||
assert _replay(same)[0].provenance == "observed_identical"
|
||||
replayed: Final = _replay(same, early)
|
||||
assert replayed == _replay(early, same)
|
||||
assert replayed[0].usage is None
|
||||
assert replayed[1].provenance == "modeled"
|
||||
assert replayed[1].usage is not None and replayed[1].usage.prompt_tokens_details.cached_tokens == 6000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ttl", [300, 3600])
|
||||
def test_prefix_match_expiry_and_usage_pricing_fields(ttl: int) -> None:
|
||||
plan: Final = CountedPromptCachePlan(6200, (_marker(ttl=ttl),))
|
||||
first: Final = _observation("first", baseline_equivalent=True, plan=plan)
|
||||
# Each replay starts from the original observation, so warm does not refresh the expiry case.
|
||||
warm: Final = _replay(first, _observation("warm", 10000.0 + ttl - 0.01, plan=plan))[-1]
|
||||
cold: Final = _replay(first, _observation("cold", 10000.0 + ttl, plan=plan))[-1]
|
||||
assert warm.reason == "cache_prefix_available" and cold.reason == "cache_prefix_expired"
|
||||
assert warm.usage is not None and cold.usage is not None
|
||||
assert warm.usage.prompt_tokens_details.cached_tokens == 6000
|
||||
assert cold.usage.prompt_tokens_details.cached_tokens == 0
|
||||
assert cold.usage.prompt_tokens_details.cache_creation_tokens == 6000
|
||||
unaffected: Final = {"prompt_tokens", "total_tokens", "prompt_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens"}
|
||||
assert warm.usage.model_dump(exclude=unaffected) == first.usage.model_dump(exclude=unaffected)
|
||||
assert cold.usage.model_dump(exclude=unaffected) == first.usage.model_dump(exclude=unaffected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("warm_tail", (False, True))
|
||||
def test_growth_lookback_and_mixed_ttl_keep_distinct_read_write_buckets(warm_tail: bool) -> None:
|
||||
first: Final = _observation("first", baseline_equivalent=True)
|
||||
grown: Final = CountedPromptCachePlan(7100, (_marker("grown", 3600, 6500, ("prefix",)), _marker("tail", 300, 7000)))
|
||||
# Initial unseen suffixes remain unknown within their potential pre-existing cache horizon.
|
||||
second: Final = _replay(first, _observation("second", 10001.0, plan=grown))[-1]
|
||||
assert second.reason == "history_unavailable"
|
||||
history: Final = BaselineHistory(
|
||||
first_at=1.0, last_at=10000.0, equivalent=False, uncertain_before=1.0,
|
||||
entries=(CacheEntry("tail:300", "tail", 7000, 300, 10000.0, 10300.0),) if warm_tail else (),
|
||||
)
|
||||
_, estimates = advance_baseline_history(history, (_observation("mixed", 10001.0, plan=grown),))
|
||||
usage: Final = estimates[0].usage
|
||||
assert usage is not None
|
||||
assert usage.prompt_tokens_details.text_tokens == 100
|
||||
# Anthropic billing locations: B is the highest 1h breakpoint AFTER the highest hit A.
|
||||
# https://platform.claude.com/docs/en/build-with-claude/prompt-caching#mixing-different-ttls (2026-09-15)
|
||||
assert usage.prompt_tokens_details.cached_tokens == (7000 if warm_tail else 0)
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens == (0 if warm_tail else 6500)
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_5m_input_tokens == (0 if warm_tail else 500)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["prefix", "ttl", "unavailable", "failed", "response_cache"])
|
||||
def test_uncertainty_and_replays_do_not_manufacture_hits(change: str) -> None:
|
||||
first: Final = _observation("first", baseline_equivalent=True)
|
||||
changes: Final = {
|
||||
"prefix": {"plan": CountedPromptCachePlan(6200, (_marker("changed"),))},
|
||||
"ttl": {"plan": CountedPromptCachePlan(6200, (_marker(ttl=300),))},
|
||||
"unavailable": {"plan": None, "reason": "token_count_unavailable"},
|
||||
"failed": {"outcome": "uncertain", "reason": "incomplete_response"},
|
||||
"response_cache": {"outcome": "response_cache"},
|
||||
}
|
||||
second: Final = _observation("second", 10001.0, **changes[change])
|
||||
third: Final = _observation("third", 10002.0)
|
||||
middle, result = _replay(first, second, third)[1:]
|
||||
assert middle.usage is None
|
||||
if change in ("unavailable", "failed", "ttl"):
|
||||
assert result.usage is None
|
||||
else:
|
||||
assert result.usage is not None and result.usage.prompt_tokens_details.cached_tokens == 6000
|
||||
|
||||
|
||||
def test_first_token_availability_and_simultaneous_divergence_are_conservative() -> None:
|
||||
slow: Final = _observation("slow", available_at=10002.0, baseline_equivalent=True)
|
||||
overlap: Final = _observation("overlap", 10001.0)
|
||||
assert _replay(slow, overlap)[-1].usage is None
|
||||
assert all(item.provenance != "observed_identical" for item in _replay(slow, _observation("tie")))
|
||||
|
||||
|
||||
def test_invalid_usage_and_invalid_count_plan_cannot_seed_cache() -> None:
|
||||
bad: Final = _observation("bad", baseline_equivalent=True, usage=_usage().model_copy(update={"total_tokens": 1}))
|
||||
assert all(item.usage is None for item in _replay(bad, _observation("next", 10001.0)))
|
||||
broken: Final = CountedPromptCachePlan(6200, (replace(_marker(), prefix_tokens=7000),))
|
||||
assert _replay(_observation("bad", plan=broken))[0].usage is None
|
||||
|
||||
|
||||
def test_overlapping_uncertain_request_cannot_be_warmed_by_a_later_callback() -> None:
|
||||
uncertain: Final = _observation("incomplete", outcome="uncertain", available_at=10010.0)
|
||||
overlap: Final = _observation("overlap", 10001.0)
|
||||
during: Final = _observation("during", 10002.0)
|
||||
after: Final = _observation("after", 10011.0)
|
||||
warmed: Final = _observation("warmed", 10012.0)
|
||||
estimates: Final = _replay(uncertain, overlap, during, after, warmed)
|
||||
assert estimates[1].reason == estimates[2].reason == "concurrent_uncertainty"
|
||||
assert estimates[3].usage is None
|
||||
assert estimates[4].usage is not None and estimates[4].usage.prompt_tokens_details.cached_tokens == 6000
|
||||
|
||||
|
||||
def test_modeled_read_cannot_recharge_the_original_private_write_count() -> None:
|
||||
warm: Final = _replay(_observation("initial", baseline_equivalent=True), _observation("warm", 10001.0))[-1]
|
||||
assert warm.usage is not None
|
||||
prices: Final = {
|
||||
**litellm.get_model_info("claude-opus-5", custom_llm_provider="anthropic"),
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 1e-7,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"provider_specific_entry": {"fast": 2.0, "us": 1.1},
|
||||
}
|
||||
input_cost, output_cost = cost_per_token("claude-opus-5", warm.usage, model_info=prices)
|
||||
assert input_cost + output_cost == pytest.approx((200 * 1e-6 + 6000 * 1e-7 + 30 * 2e-6) * 2.0 * 1.1)
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -23,13 +23,13 @@ pytestmark = pytest.mark.usefixtures("local_model_cost_map")
|
|||
def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], continuing: bool) -> None:
|
||||
usage: Final = _usage(1000, 0, 1000, 100).model_copy(update=modifier)
|
||||
expected: Final = (_usage(1000, 1000, 0, 100) if continuing else usage).model_copy(update=modifier)
|
||||
normalized: Final = _baseline_usage(usage, continuing)
|
||||
normalized: Final = _baseline_usage(expected)
|
||||
cache_fields: Final = {"prompt_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens"}
|
||||
assert normalized.model_dump(exclude=cache_fields) == usage.model_dump(exclude=cache_fields)
|
||||
assert usage.prompt_tokens_details.cached_tokens == 0
|
||||
selected_cost: Final = 0.013
|
||||
assert compute_autorouter_savings(
|
||||
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, conversation_continuing=continuing,
|
||||
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_usage=expected,
|
||||
cost_breakdown={"input_cost": 0.01, "output_cost": 0.003},
|
||||
) == pytest.approx(sum(anthropic_cost_per_token("claude-opus-5", expected)) - selected_cost)
|
||||
|
||||
|
|
@ -41,7 +41,7 @@ def test_anthropic_baseline_keeps_negotiated_prices_with_provider_multiplier() -
|
|||
}
|
||||
usage: Final = _usage(1000, 1000, 0, 100).model_copy(update={"speed": "fast"})
|
||||
assert compute_autorouter_savings(
|
||||
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info,
|
||||
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info, baseline_usage=usage,
|
||||
cost_breakdown={"input_cost": 0.01, "output_cost": 0.003},
|
||||
) == pytest.approx(0.0015 * 2 - 0.013)
|
||||
|
||||
|
|
@ -405,146 +405,109 @@ def test_negative_token_counts_clamp_to_zero():
|
|||
assert result.prompt_caching == 0.0
|
||||
|
||||
|
||||
def _usage(fresh: int, cached: int, written: int, out: int) -> Usage:
|
||||
def _usage(fresh: int, cached: int, written: int, out: int, *, hour: bool = False, image: int = 0) -> Usage:
|
||||
"""Usage as the spend log records it; `prompt_tokens` is the inclusive total."""
|
||||
return Usage(
|
||||
prompt_tokens=fresh + cached + written,
|
||||
completion_tokens=out,
|
||||
total_tokens=fresh + cached + written + out,
|
||||
prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh},
|
||||
prompt_tokens_details={
|
||||
"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh - image, "image_tokens": image,
|
||||
"cache_creation_token_details": {"ephemeral_1h_input_tokens": written} if hour else None,
|
||||
},
|
||||
cache_read_input_tokens=cached,
|
||||
cache_creation_input_tokens=written,
|
||||
)
|
||||
|
||||
|
||||
def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True) -> float:
|
||||
"""Savings for a request, defaulting to a conversation already underway.
|
||||
|
||||
`continuing=True` is the mid-conversation case, where the baseline had the prompt
|
||||
cached and this request's write is what the switch cost. `continuing=False` is a
|
||||
conversation's first turn, where nothing was cached for any model.
|
||||
"""
|
||||
def _savings(baseline: str, selected: str, usage: Usage, baseline_usage: Usage | None = None) -> float | None:
|
||||
return compute_autorouter_savings(
|
||||
baseline_model=baseline,
|
||||
selected_model=selected,
|
||||
selected_provider="anthropic",
|
||||
usage=usage,
|
||||
conversation_continuing=continuing,
|
||||
baseline_usage=baseline_usage,
|
||||
)
|
||||
|
||||
|
||||
def test_switching_models_mid_conversation_charges_the_cold_cache_write():
|
||||
"""Staying on one model writes the cache once and reads it thereafter. Switching
|
||||
leaves the new model cold, so it pays to write the whole prompt again; when that
|
||||
charge outweighs the cheaper rates the route lost money and must report a loss.
|
||||
|
||||
Pricing the baseline as if it too re-wrote the cache credits a charge it never
|
||||
paid, which is how a losing switch used to read as the largest saving on the page.
|
||||
"""
|
||||
usage = _usage(fresh=3, cached=500, written=12304, out=500)
|
||||
result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage)
|
||||
|
||||
sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic")
|
||||
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
|
||||
warm_baseline = (
|
||||
3 * sonnet["input_cost_per_token"]
|
||||
+ 12804 * sonnet["cache_read_input_token_cost"]
|
||||
+ 500 * sonnet["output_cost_per_token"]
|
||||
@pytest.mark.parametrize("baseline, selected, actual, modeled, loses_money", [
|
||||
pytest.param("claude-sonnet-5", "claude-haiku-4-5", _usage(3, 500, 12304, 500),
|
||||
_usage(3, 12804, 0, 500), True, id="warm-baseline-cold-route"),
|
||||
pytest.param("claude-opus-5", "claude-opus-5", _usage(0, 0, 20000, 1000),
|
||||
_usage(0, 20000, 0, 1000), True, id="same-model-cold-route"),
|
||||
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(0, 19000, 1000, 1000),
|
||||
_usage(0, 19500, 500, 1000), False, id="partly-cached-growth"),
|
||||
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(0, 0, 100000, 1000, hour=True),
|
||||
_usage(0, 0, 100000, 1000, hour=True), False, id="expired-one-hour"),
|
||||
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(0, 0, 100000, 1000, hour=True),
|
||||
_usage(0, 100000, 0, 1000), True, id="invented-one-hour-hit"),
|
||||
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(4000, 0, 16000, 1000, hour=True, image=4000),
|
||||
_usage(4000, 0, 16000, 1000, hour=True, image=4000), False, id="image-and-one-hour-write"),
|
||||
])
|
||||
def test_supplied_baseline_usage_is_priced_independently(
|
||||
baseline: str, selected: str, actual: Usage, modeled: Usage, loses_money: bool,
|
||||
) -> None:
|
||||
result: Final = _savings(baseline, selected, actual, modeled)
|
||||
expected: Final = sum(generic_cost_per_token(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum(
|
||||
generic_cost_per_token(model=selected, usage=actual, custom_llm_provider="anthropic")
|
||||
)
|
||||
actually_paid = (
|
||||
3 * haiku["input_cost_per_token"]
|
||||
+ 500 * haiku["cache_read_input_token_cost"]
|
||||
+ 12304 * haiku["cache_creation_input_token_cost"]
|
||||
+ 500 * haiku["output_cost_per_token"]
|
||||
)
|
||||
assert result == pytest.approx(warm_baseline - actually_paid)
|
||||
assert result < 0, "a cache-thrashing switch must report a loss, not a saving"
|
||||
|
||||
phantom = 12304 * sonnet["cache_creation_input_token_cost"]
|
||||
assert result != pytest.approx(warm_baseline + phantom - actually_paid)
|
||||
assert result == pytest.approx(expected)
|
||||
assert result is not None and (result < 0) is loses_money
|
||||
assert _baseline_usage(modeled).prompt_tokens_details == modeled.prompt_tokens_details
|
||||
|
||||
|
||||
def test_a_cold_switch_never_beats_turning_caching_off():
|
||||
"""Switching to a cold model makes it write the whole prompt again. That write is a
|
||||
real cost of switching, so the same traffic must look worse than if caching were off
|
||||
entirely.
|
||||
|
||||
The baseline is priced as a warm cache even though this request read nothing: a
|
||||
switch reads nothing precisely because the new model's cache is empty, and staying
|
||||
on one model would have had the prompt cached already. Gating the warm baseline on
|
||||
a read charged the baseline a write it would never repeat, which made a cold switch
|
||||
report a larger saving than no caching at all.
|
||||
"""
|
||||
cold_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000))
|
||||
caching_off = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(20_000, 0, 0, 1_000))
|
||||
|
||||
assert cold_switch < caching_off
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
|
||||
warm_baseline = 20_000 * opus["cache_read_input_token_cost"] + 1_000 * opus["output_cost_per_token"]
|
||||
actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"]
|
||||
assert cold_switch == pytest.approx(warm_baseline - actually_paid)
|
||||
@pytest.mark.parametrize("modifier, multiplier", [({}, 1.0), ({"inference_geo": "us"}, 1.1), ({"speed": "fast"}, 2.0)])
|
||||
@pytest.mark.parametrize("negotiated", [False, True])
|
||||
@pytest.mark.parametrize("provenance", [None, "modeled", "observed_initial"])
|
||||
def test_observed_initial_uses_provider_billing_and_effective_rates(
|
||||
modifier: dict[str, str], multiplier: float, negotiated: bool,
|
||||
provenance: Literal["modeled", "observed_initial"] | None,
|
||||
) -> None:
|
||||
usage: Final = _usage(1000, 2000, 3000, 100).model_copy(update=modifier)
|
||||
info: Final = litellm.get_model_info("claude-opus-5", "anthropic").copy()
|
||||
if negotiated:
|
||||
info["input_cost_per_token"] = 1e-6
|
||||
info["output_cost_per_token"] = 2e-6
|
||||
info["cache_read_input_token_cost"] = 3e-7
|
||||
info["cache_creation_input_token_cost"] = 4e-6
|
||||
billed: Final = anthropic_cost_per_token("claude-opus-5", usage, model_info=info)
|
||||
if negotiated:
|
||||
assert sum(billed) == pytest.approx(0.0138 * multiplier)
|
||||
assert compute_autorouter_savings(
|
||||
"anthropic/claude-opus-5", "claude-opus-5", "anthropic", usage,
|
||||
selected_info=info, baseline_info=info, baseline_usage=usage,
|
||||
baseline_deployment_id="same", selected_deployment_id="same",
|
||||
cost_breakdown={"input_cost": billed[0], "output_cost": billed[1]},
|
||||
baseline_provenance=provenance,
|
||||
) == 0.0
|
||||
|
||||
|
||||
def test_moving_one_token_between_cache_buckets_does_not_move_the_answer():
|
||||
"""A continuing conversation writes a few new tokens and reads the rest. Treating the
|
||||
presence of a write as the signal for a switch made that ordinary increment flip the
|
||||
result, so a request reading 19,999 and writing 1 landed somewhere entirely different
|
||||
from one reading 20,000 and writing none.
|
||||
"""
|
||||
reads_nothing = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000))
|
||||
reads_one = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 1, 19_999, 1_000))
|
||||
assert reads_one == pytest.approx(reads_nothing, abs=1e-4)
|
||||
|
||||
|
||||
def test_multimodal_prompts_are_priced_on_the_baseline_too():
|
||||
"""The baseline is this same request met by a warm cache, so every field it was
|
||||
priced on has to survive. Rebuilding the details from the cache buckets alone
|
||||
dropped the image and audio counts, which priced the baseline as a text-only
|
||||
request that never ran and shrank the reported saving on multimodal traffic.
|
||||
"""
|
||||
details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000}
|
||||
with_images = Usage(
|
||||
prompt_tokens=20_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=21_000,
|
||||
prompt_tokens_details=details,
|
||||
)
|
||||
baseline = _baseline_usage(with_images, conversation_continuing=True)
|
||||
|
||||
assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline"
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
|
||||
text_only = 20_000 * opus["cache_read_input_token_cost"]
|
||||
assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving"
|
||||
|
||||
|
||||
def test_the_baseline_is_never_charged_a_cache_write():
|
||||
"""Carrying the details through must not carry the 5m/1h creation breakdown with
|
||||
them. `generic_cost_per_token` charges a creation cost whenever that breakdown is
|
||||
present, even against a zeroed creation count, which would put the phantom write
|
||||
back on the baseline for every long-cache request.
|
||||
"""
|
||||
long_cache = Usage(
|
||||
prompt_tokens=20_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=21_000,
|
||||
prompt_tokens_details={
|
||||
"cached_tokens": 0,
|
||||
"cache_creation_tokens": 20_000,
|
||||
"text_tokens": 0,
|
||||
"cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000},
|
||||
},
|
||||
)
|
||||
baseline = _baseline_usage(long_cache, conversation_continuing=True)
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
|
||||
assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), (
|
||||
"the baseline reads a warm cache; it never pays to create one"
|
||||
)
|
||||
@pytest.mark.parametrize("model, deployment, known, delta", [
|
||||
("claude-sonnet-5", "same", "observed", 0.0),
|
||||
("claude-opus-5", "other", "observed", 0.0),
|
||||
("claude-opus-5", "", "observed", 0.0),
|
||||
("claude-opus-5", "same", "missing", 0.0),
|
||||
("claude-opus-5", "same", "different", 0.0),
|
||||
("claude-opus-5", "same", "observed", 0.01),
|
||||
("claude-opus-5", "same", "prices", 0.0),
|
||||
("claude-opus-5", "same", "unbilled", 0.0),
|
||||
])
|
||||
def test_initial_provenance_cannot_override_mismatched_evidence(
|
||||
model: str, deployment: str, known: Literal["observed", "missing", "different", "prices", "unbilled"], delta: float,
|
||||
) -> None:
|
||||
usage: Final = _usage(1000, 0, 1000, 100)
|
||||
billed: Final = anthropic_cost_per_token("claude-opus-5", usage)
|
||||
info: Final = litellm.get_model_info(model, "anthropic").copy()
|
||||
if known == "prices":
|
||||
info["cache_read_input_token_cost"] = 0.001 # No reads here: equal charge alone cannot establish equal rates.
|
||||
assert compute_autorouter_savings(
|
||||
"claude-opus-5", model, "anthropic", usage,
|
||||
baseline_usage=(None if known == "missing" else _usage(1000, 1000, 0, 100) if known == "different" else usage),
|
||||
selected_info=info,
|
||||
baseline_provenance="observed_initial",
|
||||
baseline_deployment_id="same", selected_deployment_id=deployment,
|
||||
cost_breakdown=None if known == "unbilled" else {"input_cost": billed[0] + delta, "output_cost": billed[1]},
|
||||
) is None
|
||||
|
||||
|
||||
def test_uncached_request_is_the_plain_rate_difference():
|
||||
|
|
@ -565,11 +528,12 @@ def test_escalation_reports_its_real_cost():
|
|||
|
||||
|
||||
def test_autorouter_savings_zero_when_model_unchanged():
|
||||
assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0
|
||||
usage: Final = _usage(3, 500, 12304, 500)
|
||||
assert _savings("claude-opus-5", "claude-opus-5", usage, usage) == 0.0
|
||||
|
||||
|
||||
def test_autorouter_savings_unknown_baseline_fails_open_to_zero():
|
||||
assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0
|
||||
def test_autorouter_savings_unknown_baseline_remains_unknown():
|
||||
assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) is None
|
||||
|
||||
|
||||
def test_autorouter_savings_zero_without_baseline():
|
||||
|
|
@ -584,9 +548,7 @@ def test_autorouter_savings_zero_without_baseline():
|
|||
assert result.autorouter == 0.0
|
||||
|
||||
|
||||
def test_compute_savings_spend_carries_a_losing_switch_through():
|
||||
"""The signed value must survive into SavingsSpend; clamping it here would put the
|
||||
dashboard back to only ever showing gains."""
|
||||
def test_compute_savings_spend_carries_a_recorded_losing_switch_through():
|
||||
result = compute_savings_spend(
|
||||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
|
|
@ -594,6 +556,7 @@ def test_compute_savings_spend_carries_a_losing_switch_through():
|
|||
gateway_injected_cache=True,
|
||||
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-sonnet-5"},
|
||||
usage_object=_cached_usage_object(),
|
||||
recorded_autorouter_savings=-0.01,
|
||||
)
|
||||
assert result.autorouter < 0
|
||||
|
||||
|
|
@ -628,18 +591,10 @@ def test_malformed_usage_object_does_not_fail_the_spend_write():
|
|||
assert result.compression > 0
|
||||
|
||||
|
||||
def test_the_same_deployment_spelled_two_ways_is_not_a_switch():
|
||||
"""The spend log records a normalized model name while the baseline arrives as the
|
||||
operator wrote it in config. Comparing the raw strings makes a request that never
|
||||
changed model look like a switch, and prices one deployment against itself."""
|
||||
# Must be a cached request: the baseline arm is priced against a warm cache and the
|
||||
# selected arm against what was actually paid, so treating one deployment as two
|
||||
# charges it a cold-cache write it never took, inventing a loss on a request that
|
||||
# never changed model. An uncached request prices identically either way and would
|
||||
# make this assertion vacuous.
|
||||
usage = _usage(fresh=3, cached=500, written=12304, out=500)
|
||||
assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0
|
||||
assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0
|
||||
def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None:
|
||||
usage: Final = _usage(3, 500, 12304, 500)
|
||||
assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage, usage) == 0.0
|
||||
assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0
|
||||
|
||||
|
||||
def test_baseline_is_priced_under_its_own_provider():
|
||||
|
|
@ -663,99 +618,9 @@ def test_baseline_is_priced_under_its_own_provider():
|
|||
assert azure > 0 > deepseek
|
||||
|
||||
|
||||
def test_unresolvable_baseline_fails_open_to_zero():
|
||||
def test_unresolvable_baseline_remains_unknown():
|
||||
usage = _usage(fresh=2000, cached=0, written=0, out=500)
|
||||
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0
|
||||
|
||||
|
||||
def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty():
|
||||
"""Nothing was cached anywhere on a conversation's first turn, so the baseline would
|
||||
have paid the same cache write. Charging it to the selected arm alone reported a
|
||||
fraction of the real saving; on this shape roughly 4% of it.
|
||||
"""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False)
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
|
||||
both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - (
|
||||
20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"]
|
||||
)
|
||||
assert first_turn == pytest.approx(both_write)
|
||||
|
||||
mid_conversation = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)
|
||||
assert first_turn > mid_conversation * 10, "a first turn must not be priced as a switch"
|
||||
|
||||
|
||||
def test_a_first_turn_that_saves_money_never_reports_a_loss():
|
||||
"""The write premium is fixed by prompt size while the saving grows with completion
|
||||
length, so charging the write to a first turn made short answers over a large cached
|
||||
prompt read as losses on requests that genuinely saved. That is the shape most likely
|
||||
to be on the dashboard, and the sign has to be right.
|
||||
"""
|
||||
short_answer = _usage(fresh=0, cached=0, written=20_000, out=200)
|
||||
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, continuing=False) > 0
|
||||
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer) < 0
|
||||
|
||||
|
||||
def test_an_undetermined_conversation_shape_stays_conservative():
|
||||
"""The default must charge the write. A caller that cannot be read, or a surface the
|
||||
router never classified, has said nothing about whether the baseline was warm, and a
|
||||
savings figure must not inflate on a guess.
|
||||
"""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
defaulted = compute_autorouter_savings(
|
||||
baseline_model="anthropic/claude-opus-5",
|
||||
selected_model="claude-haiku-4-5",
|
||||
selected_provider="anthropic",
|
||||
usage=usage,
|
||||
)
|
||||
assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage))
|
||||
assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False)
|
||||
|
||||
|
||||
def test_a_continuing_turn_on_the_same_model_writes_its_growth_on_both_arms():
|
||||
"""A conversation that grew by a few tokens writes those on whatever model serves
|
||||
it, and they are new to every model, so the baseline would have written them too.
|
||||
Moving them into the baseline's read bucket forgives it a write it really owes and
|
||||
shrinks the reported saving on ordinary steady-state traffic.
|
||||
"""
|
||||
usage = _usage(fresh=0, cached=19_900, written=100, out=1_000)
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
|
||||
|
||||
def cost(info: dict) -> float:
|
||||
return (
|
||||
19_900 * info["cache_read_input_token_cost"]
|
||||
+ 100 * info["cache_creation_input_token_cost"]
|
||||
+ 1_000 * info["output_cost_per_token"]
|
||||
)
|
||||
|
||||
both_write_the_growth = cost(opus) - cost(haiku)
|
||||
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) == pytest.approx(both_write_the_growth)
|
||||
|
||||
|
||||
def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write():
|
||||
"""A model holding a small prefix of this prompt still has to write the rest, and
|
||||
that write is the switch's cost. Keying the same-model case off reading *anything*
|
||||
rather than reading *most of it* would hand this request the full rate gap and
|
||||
inflate the saving by an order of magnitude.
|
||||
"""
|
||||
mostly_written = _usage(fresh=0, cached=500, written=19_500, out=1_000)
|
||||
reported = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", mostly_written)
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
|
||||
if_treated_as_same_model = (
|
||||
500 * opus["cache_read_input_token_cost"]
|
||||
+ 19_500 * opus["cache_creation_input_token_cost"]
|
||||
+ 1_000 * opus["output_cost_per_token"]
|
||||
) - (
|
||||
500 * haiku["cache_read_input_token_cost"]
|
||||
+ 19_500 * haiku["cache_creation_input_token_cost"]
|
||||
+ 1_000 * haiku["output_cost_per_token"]
|
||||
)
|
||||
assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation"
|
||||
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) is None
|
||||
|
||||
|
||||
def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt():
|
||||
|
|
@ -771,7 +636,7 @@ def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt():
|
|||
selected_model="claude-haiku-4-5",
|
||||
selected_provider="anthropic",
|
||||
usage=first_turn,
|
||||
conversation_continuing=False,
|
||||
baseline_usage=first_turn,
|
||||
)
|
||||
|
||||
gpt5 = litellm.get_model_info("gpt-5", "openai")
|
||||
|
|
@ -807,7 +672,7 @@ def _priced_chat_model_without_cache_read_rate() -> tuple[str, str, str]:
|
|||
usage=_usage(fresh=1_000, cached=0, written=0, out=100),
|
||||
conversation_continuing=True,
|
||||
)
|
||||
if priced == 0.0:
|
||||
if priced is None or priced == 0.0:
|
||||
continue
|
||||
return key, key.removeprefix(f"{provider}/"), provider
|
||||
raise AssertionError("the bundled map has no per-token chat model without a cache-read rate")
|
||||
|
|
@ -825,7 +690,7 @@ def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate():
|
|||
selected_model="claude-haiku-4-5",
|
||||
selected_provider="anthropic",
|
||||
usage=continuing,
|
||||
conversation_continuing=True,
|
||||
baseline_usage=_usage(0, 20000, 0, 1000),
|
||||
)
|
||||
|
||||
baseline = litellm.get_model_info(baseline_name, baseline_provider)
|
||||
|
|
@ -966,7 +831,7 @@ def test_a_baseline_recorded_on_the_decision_turns_the_driver_on():
|
|||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=True,
|
||||
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"},
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
)
|
||||
assert result.autorouter != 0.0
|
||||
|
||||
|
|
@ -981,13 +846,13 @@ def test_a_leftover_configured_baseline_does_not_override_the_recorded_one(monke
|
|||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=True,
|
||||
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"},
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
)
|
||||
against_opus = compute_autorouter_savings(
|
||||
baseline_model="anthropic/claude-opus-5",
|
||||
selected_model="claude-haiku-4-5",
|
||||
selected_provider="anthropic",
|
||||
usage=Usage(**_cached_usage_object()),
|
||||
usage=_usage(12807, 0, 0, 500),
|
||||
)
|
||||
assert result.autorouter == against_opus
|
||||
|
||||
|
|
@ -1054,12 +919,12 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
|
|||
("baseline", "selected", 2.0, None, 0.0, -0.015),
|
||||
("baseline", "selected", 1.0, None, 0.0, 0.0),
|
||||
("baseline", "selected", 0.1, 0.004, 0.001, 0.01),
|
||||
("baseline", "baseline", 0.1, 0.004, 0.001, -0.001),
|
||||
(None, "selected", 0.1, None, 0.0, 0.0),
|
||||
("baseline", None, 0.1, None, 0.0, 0.0),
|
||||
("baseline", "baseline", 0.1, 0.004, 0.001, 0.01),
|
||||
(None, "selected", 0.1, None, 0.0, 0.006),
|
||||
("baseline", None, 0.1, None, 0.0, 0.0075),
|
||||
(None, None, 0.1, None, 0.0, 0.0),
|
||||
("", "selected", 0.1, None, 0.0, 0.0),
|
||||
("baseline", "", 0.1, None, 0.0, 0.0),
|
||||
("", "selected", 0.1, None, 0.0, 0.006),
|
||||
("baseline", "", 0.1, None, 0.0, 0.0075),
|
||||
],
|
||||
)
|
||||
def test_autorouter_savings_distinguishes_priced_deployments(
|
||||
|
|
@ -1172,7 +1037,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
|
|||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=True,
|
||||
routing_decision=decision,
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
llm_router=lambda: router,
|
||||
)
|
||||
at_public_rate = compute_savings_spend(
|
||||
|
|
@ -1181,7 +1046,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
|
|||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=True,
|
||||
routing_decision={k: v for k, v in decision.items() if k != "savings_baseline_deployment_id"},
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
llm_router=lambda: router,
|
||||
)
|
||||
assert with_deployment_rate.autorouter > at_public_rate.autorouter
|
||||
|
|
@ -1234,9 +1099,8 @@ def test_a_boolean_is_not_a_recorded_savings_figure():
|
|||
assert result.autorouter == 0.0
|
||||
|
||||
|
||||
def test_rows_written_before_the_field_shipped_recompute():
|
||||
"""No recorded figure means the row predates the logging-path stamp; the writer
|
||||
recomputes exactly what the one shared helper would have recorded."""
|
||||
@pytest.mark.parametrize("continuing", [False, True])
|
||||
def test_legacy_cache_rows_without_an_estimate_do_not_invent_a_new_figure(continuing: bool) -> None:
|
||||
from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request
|
||||
|
||||
recomputed = compute_savings_spend(
|
||||
|
|
@ -1244,17 +1108,17 @@ def test_rows_written_before_the_field_shipped_recompute():
|
|||
custom_llm_provider="anthropic",
|
||||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=False,
|
||||
routing_decision=_routed_decision(),
|
||||
routing_decision={**_routed_decision(), "conversation_continuing": continuing},
|
||||
usage_object=_cached_usage_object(),
|
||||
)
|
||||
direct = autorouter_savings_for_request(
|
||||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
routing_decision=_routed_decision(),
|
||||
routing_decision={**_routed_decision(), "conversation_continuing": continuing},
|
||||
usage_object=_cached_usage_object(),
|
||||
)
|
||||
assert direct is not None and direct != 0.0
|
||||
assert recomputed.autorouter == direct
|
||||
assert direct is None
|
||||
assert recomputed.autorouter == 0.0
|
||||
|
||||
|
||||
def test_driver_off_is_none_not_zero_for_the_request_helper():
|
||||
|
|
@ -1294,7 +1158,7 @@ def test_logging_payload_never_stamps_internal_calls():
|
|||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
model_id=None,
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
cost_breakdown=None,
|
||||
)
|
||||
assert stamped is not None and stamped != 0.0
|
||||
|
|
@ -1304,7 +1168,7 @@ def test_logging_payload_never_stamps_internal_calls():
|
|||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
model_id=None,
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
cost_breakdown=None,
|
||||
)
|
||||
assert internal is None
|
||||
|
|
@ -1320,13 +1184,13 @@ def test_savings_are_net_of_a_priced_classifier():
|
|||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
routing_decision=_routed_decision(),
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
)
|
||||
net = autorouter_savings_for_request(
|
||||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
routing_decision={**_routed_decision(), "classifier_cost": 0.005},
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
)
|
||||
assert gross is not None and net == pytest.approx(gross - 0.005)
|
||||
|
||||
|
|
@ -1339,13 +1203,13 @@ def test_an_unpriced_classifier_deducts_nothing(classifier_cost: object):
|
|||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
routing_decision=_routed_decision(),
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
)
|
||||
with_cost_field = autorouter_savings_for_request(
|
||||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
routing_decision={**_routed_decision(), "classifier_cost": classifier_cost},
|
||||
usage_object=_cached_usage_object(),
|
||||
usage_object=_usage(12807, 0, 0, 500).model_dump(),
|
||||
)
|
||||
assert with_cost_field == gross
|
||||
|
||||
|
|
@ -1454,4 +1318,41 @@ def test_marks_gateway_injection_credits_only_the_deployment_that_was_injected()
|
|||
assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, "dep-a") is True
|
||||
assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, None) is True
|
||||
assert marks_gateway_injection({"litellm_call_id": "c1"}, "dep-a") is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("classifier", [0.0, 0.02])
|
||||
def test_observed_baseline_keeps_both_costs_and_classifier_overhead(classifier: float) -> None:
|
||||
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot, price_baseline_comparison
|
||||
|
||||
snapshot: Final = BaselineCostSnapshot(
|
||||
model="baseline", provider="anthropic", prices=None,
|
||||
actual_spend=0.17, classifier_cost=classifier,
|
||||
)
|
||||
restored: Final = BaselineCostSnapshot.model_validate_json(snapshot.model_dump_json())
|
||||
result: Final = price_baseline_comparison(restored, Usage(prompt_tokens=100, completion_tokens=10), "observed_identical")
|
||||
assert result is not None
|
||||
assert result.baseline == snapshot.actual_spend
|
||||
assert result.actual == snapshot.actual_spend + classifier
|
||||
assert result.savings == pytest.approx(-classifier)
|
||||
assert price_baseline_comparison(restored, None, None) is None
|
||||
|
||||
|
||||
def test_modeled_baseline_uses_recorded_prices_and_preserves_other_actual_charges() -> None:
|
||||
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot, price_baseline_comparison
|
||||
|
||||
snapshot: Final = BaselineCostSnapshot(
|
||||
model="claude-opus-5", provider="anthropic",
|
||||
prices={
|
||||
**litellm.get_model_info("claude-opus-5", custom_llm_provider="anthropic"),
|
||||
"input_cost_per_token": 0.001, "output_cost_per_token": 0.002,
|
||||
},
|
||||
actual_token_cost=0.2, actual_spend=0.23, classifier_cost=0.01,
|
||||
)
|
||||
usage: Final = Usage(prompt_tokens=100, completion_tokens=10)
|
||||
result: Final = price_baseline_comparison(snapshot, usage, "modeled")
|
||||
assert result is not None
|
||||
assert result.actual == pytest.approx(0.24)
|
||||
assert result.baseline == pytest.approx(0.12 + 0.03)
|
||||
assert result.savings == pytest.approx(-0.09)
|
||||
assert price_baseline_comparison(snapshot.model_copy(update={"prices": None}), usage, "modeled") is None
|
||||
assert marks_gateway_injection({"litellm_gateway_injected_cache": True}, "dep-a") is False
|
||||
|
|
|
|||
|
|
@ -3745,7 +3745,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "azure_spillover": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
@ -3841,7 +3841,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
@ -3935,7 +3935,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import datetime
|
|||
import json
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -5208,3 +5209,17 @@ def test_azure_spillover_absent_without_spillover_headers():
|
|||
)
|
||||
metadata = json.loads(payload["metadata"])
|
||||
assert metadata["azure_spillover"] is None
|
||||
|
||||
|
||||
def test_baseline_estimate_metadata_comes_from_the_logging_stamp() -> None:
|
||||
supplied: Final = MappingProxyType({"version": 1, "status": "estimated", "reason": "caller_supplied"})
|
||||
recorded: Final = MappingProxyType({"version": 1, "status": "unknown", "reason": "history_unavailable"})
|
||||
result: Final = _get_spend_logs_metadata(
|
||||
{"autorouter_savings": 999.0, "autorouter_savings_estimate": supplied}, # mutable-ok: legacy metadata helper accepts dicts
|
||||
autorouter_savings=None,
|
||||
autorouter_savings_estimate=recorded,
|
||||
)
|
||||
assert result["autorouter_savings"] is None
|
||||
assert result["autorouter_savings_estimate"] == recorded
|
||||
absent: Final = _get_spend_logs_metadata({"autorouter_savings_estimate": supplied}) # mutable-ok: legacy metadata helper accepts dicts
|
||||
assert absent["autorouter_savings_estimate"] is None
|
||||
|
|
|
|||
|
|
@ -3188,6 +3188,182 @@ class TestMcpToolSearchSettingsEndpoints:
|
|||
assert mock_proxy_config["save_call_count"]() == 0
|
||||
|
||||
|
||||
class TestWebSearchInterceptionSettingsEndpoints:
|
||||
@staticmethod
|
||||
def _override_auth(role: LitellmUserRoles):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="u", api_key="hashed", user_role=role
|
||||
)
|
||||
|
||||
def test_get_returns_stored_values_and_field_schema(self, mock_proxy_config, mock_auth, monkeypatch):
|
||||
import litellm
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object())
|
||||
monkeypatch.setattr(litellm, "callbacks", [WebSearchInterceptionLogger(search_tool_name="running")])
|
||||
mock_proxy_config["config"]["litellm_settings"]["websearch_interception_params"] = {
|
||||
"enabled": True,
|
||||
"enabled_providers": ["bedrock", "vertex_ai"],
|
||||
"search_tool_name": "my-perplexity-search",
|
||||
}
|
||||
|
||||
resp = client.get("/get/websearch_interception_settings")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["values"] == {
|
||||
"enabled": True,
|
||||
"enabled_providers": ["bedrock", "vertex_ai"],
|
||||
"search_tool_name": "my-perplexity-search",
|
||||
"max_agentic_loops": None,
|
||||
}
|
||||
assert resp.json()["field_schema"]["properties"]["enabled_providers"]["type"] == "array"
|
||||
|
||||
def test_update_requires_proxy_admin(self, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
|
||||
self._override_auth(LitellmUserRoles.INTERNAL_USER)
|
||||
try:
|
||||
resp = client.patch("/update/websearch_interception_settings", json={"enabled": True})
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
assert resp.status_code == 403
|
||||
assert "proxy admin" in resp.json()["detail"].lower()
|
||||
|
||||
def test_update_persists_settings(self, mock_proxy_config, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
|
||||
self._override_auth(LitellmUserRoles.PROXY_ADMIN)
|
||||
payload = {
|
||||
"enabled": True,
|
||||
"enabled_providers": ["bedrock"],
|
||||
"search_tool_name": "my-perplexity-search",
|
||||
"max_agentic_loops": 5,
|
||||
}
|
||||
try:
|
||||
resp = client.patch("/update/websearch_interception_settings", json=payload)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert mock_proxy_config["save_call_count"]() == 1
|
||||
assert mock_proxy_config["config"]["litellm_settings"]["websearch_interception_params"] == payload
|
||||
|
||||
def test_get_reports_enabled_while_the_callback_is_running_without_a_stored_flag(
|
||||
self, mock_proxy_config, mock_auth, monkeypatch
|
||||
):
|
||||
import litellm
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object())
|
||||
monkeypatch.setattr(litellm, "callbacks", [WebSearchInterceptionLogger(search_tool_name="from-config")])
|
||||
mock_proxy_config["config"]["litellm_settings"]["websearch_interception_params"] = {
|
||||
"enabled_providers": ["bedrock"],
|
||||
"search_tool_name": "my-perplexity-search",
|
||||
}
|
||||
|
||||
resp = client.get("/get/websearch_interception_settings")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["values"]["enabled"] is True
|
||||
|
||||
def test_get_reports_disabled_when_nothing_is_stored_and_nothing_is_running(
|
||||
self, mock_proxy_config, mock_auth, monkeypatch
|
||||
):
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object())
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
mock_proxy_config["config"]["litellm_settings"]["websearch_interception_params"] = {
|
||||
"enabled_providers": ["bedrock"],
|
||||
}
|
||||
|
||||
resp = client.get("/get/websearch_interception_settings")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["values"]["enabled"] is False
|
||||
assert resp.json()["values"]["enabled_providers"] == ["bedrock"]
|
||||
|
||||
def test_update_reapplies_settings_to_the_running_proxy(self, mock_proxy_config, monkeypatch):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object())
|
||||
reapply = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.proxy_config.init_websearch_interception_settings_in_db",
|
||||
reapply,
|
||||
)
|
||||
self._override_auth(LitellmUserRoles.PROXY_ADMIN)
|
||||
try:
|
||||
resp = client.patch("/update/websearch_interception_settings", json={"enabled": True})
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
reapply.assert_awaited_once()
|
||||
|
||||
def test_get_keeps_the_stored_flag_when_this_pod_has_not_reinitialized(
|
||||
self, mock_proxy_config, mock_auth, monkeypatch
|
||||
):
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object())
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
mock_proxy_config["config"]["litellm_settings"]["websearch_interception_params"] = {
|
||||
"enabled": True,
|
||||
"search_tool_name": "cluster-search",
|
||||
}
|
||||
|
||||
resp = client.get("/get/websearch_interception_settings")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["values"]["enabled"] is True
|
||||
|
||||
def test_get_reports_no_database_instead_of_empty_settings(self, mock_proxy_config, mock_auth, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
|
||||
resp = client.get("/get/websearch_interception_settings")
|
||||
|
||||
assert resp.status_code == 500, resp.text
|
||||
assert "Database not connected" in resp.json()["detail"]["error"]
|
||||
|
||||
def test_update_still_saves_when_the_live_reinit_fails(self, mock_proxy_config, monkeypatch):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object())
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.proxy_config.init_websearch_interception_settings_in_db",
|
||||
AsyncMock(side_effect=RuntimeError("callback blew up")),
|
||||
)
|
||||
self._override_auth(LitellmUserRoles.PROXY_ADMIN)
|
||||
try:
|
||||
resp = client.patch("/update/websearch_interception_settings", json={"enabled": True})
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert mock_proxy_config["save_call_count"]() == 1
|
||||
|
||||
def test_update_rejects_zero_max_agentic_loops(self, mock_proxy_config, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
|
||||
self._override_auth(LitellmUserRoles.PROXY_ADMIN)
|
||||
try:
|
||||
resp = client.patch(
|
||||
"/update/websearch_interception_settings",
|
||||
json={"enabled": True, "max_agentic_loops": 0},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
assert resp.status_code == 422
|
||||
assert mock_proxy_config["save_call_count"]() == 0
|
||||
|
||||
|
||||
def test_upload_logo_requires_proxy_admin(monkeypatch):
|
||||
"""Any authenticated key could previously write a file to the server's disk here."""
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Any, Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,7 +14,7 @@ from fastapi import HTTPException
|
|||
import litellm
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import AlertType, ProxyErrorTypes
|
||||
from litellm.proxy._types import AlertType, ProxyErrorTypes, UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
|
||||
|
|
@ -156,6 +157,23 @@ async def test_post_call_failure_hook_non_http_exception_in_callback_swallowed(
|
|||
assert out is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("logging_value", (None, "caller-controlled", {"baseline_cache_context": "untrusted"})) # mutable-ok: emulate an untrusted JSON request field
|
||||
async def test_terminal_baseline_cleanup_ignores_missing_or_untrusted_logging(
|
||||
proxy_logging: ProxyLogging, monkeypatch: pytest.MonkeyPatch, logging_value: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", ())
|
||||
proxy_logging.alert_types = [] # mutable-ok: disable optional alert sinks for this boundary test # rebind-ok: isolate the fixture-owned alert configuration
|
||||
request_data: Final = {"litellm_call_id": "untrusted-logging", "litellm_logging_obj": logging_value} # mutable-ok: the production failure owner removes internal fields in place
|
||||
result: Final = await proxy_logging.post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # exercise the existing proxy terminal owner with its legacy request dictionary contract
|
||||
request_data=request_data,
|
||||
original_exception=ValueError("original provider failure"),
|
||||
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/messages"),
|
||||
)
|
||||
assert result is None
|
||||
assert "litellm_logging_obj" not in request_data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _handle_logging_proxy_only_error
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, Final, List
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -179,6 +179,29 @@ async def _one_chunk() -> AsyncGenerator[object, None]:
|
|||
yield "chunk"
|
||||
|
||||
|
||||
class _AttributeStream:
|
||||
_hidden_params = {"model_id": "m-1"}
|
||||
model = "gpt-x"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._chunks = ("chunk-1", "chunk-2")
|
||||
self._index = 0
|
||||
self.closed = False
|
||||
|
||||
def __aiter__(self) -> "_AttributeStream":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> str:
|
||||
if self._index >= len(self._chunks):
|
||||
raise StopAsyncIteration
|
||||
chunk = self._chunks[self._index]
|
||||
self._index += 1
|
||||
return chunk
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_streaming_iterator_with_enrichment_passes_through_chunks(proxy_logging):
|
||||
async def gen():
|
||||
|
|
@ -247,6 +270,68 @@ async def test_wrap_streaming_iterator_leaves_upstream_http_exception_unattribut
|
|||
assert request_data == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_streaming_iterator_forwards_response_attributes_to_hook(proxy_logging):
|
||||
async def prefix_hook(*, response: AsyncIterator[object]) -> AsyncGenerator[object, None]:
|
||||
async for chunk in response:
|
||||
yield f"{response._hidden_params['model_id']}:{response.model}:{chunk}"
|
||||
|
||||
source = _AttributeStream()
|
||||
wrapped = proxy_logging._wrap_streaming_iterator_with_enrichment(
|
||||
callback=MagicMock(guardrail_name="g", event_hook="post_call"),
|
||||
response=source,
|
||||
hook=prefix_hook,
|
||||
request_data={},
|
||||
)
|
||||
|
||||
assert [chunk async for chunk in wrapped] == [
|
||||
"m-1:gpt-x:chunk-1",
|
||||
"m-1:gpt-x:chunk-2",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_streaming_iterator_forwards_aclose_to_upstream(proxy_logging):
|
||||
async def close_hook(*, response: AsyncIterator[object]) -> AsyncGenerator[object, None]:
|
||||
first: Final = await response.__anext__()
|
||||
yield first
|
||||
await response.aclose()
|
||||
|
||||
source = _AttributeStream()
|
||||
request_data: dict[str, object] = {}
|
||||
wrapped = proxy_logging._wrap_streaming_iterator_with_enrichment(
|
||||
callback=MagicMock(guardrail_name="g", event_hook="post_call"),
|
||||
response=source,
|
||||
hook=close_hook,
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
assert [chunk async for chunk in wrapped] == ["chunk-1"]
|
||||
assert source.closed is True
|
||||
assert request_data == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_streaming_iterator_missing_attribute_still_raises(proxy_logging):
|
||||
async def missing_attribute_hook(*, response: AsyncIterator[object]) -> AsyncGenerator[object, None]:
|
||||
_missing: Final = response.not_there
|
||||
if False:
|
||||
yield
|
||||
|
||||
request_data: dict[str, object] = {}
|
||||
wrapped = proxy_logging._wrap_streaming_iterator_with_enrichment(
|
||||
callback=MagicMock(guardrail_name="hook-bug", event_hook="post_call"),
|
||||
response=_one_chunk(),
|
||||
hook=missing_attribute_hook,
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
with pytest.raises(AttributeError):
|
||||
async for _ in wrapped:
|
||||
pass
|
||||
assert request_data["metadata"]["applied_guardrails"] == ["hook-bug"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# async_post_call_streaming_hook
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -2269,6 +2269,10 @@ class TestAutoRouterSessionRepository:
|
|||
"classifier_cost": 0.01,
|
||||
"tier_turns": {"complex": 3},
|
||||
"baseline_models": {"anthropic/claude-opus-5": 3},
|
||||
"savings_estimated_turns": 3,
|
||||
"savings_estimated_actual_spend": 0.14,
|
||||
"savings_estimated_saved_spend": 0.24,
|
||||
"savings_estimated_baseline_models": {"anthropic/claude-opus-5": 3},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -2295,6 +2299,9 @@ class TestAutoRouterSessionRepository:
|
|||
assert (row.router_name, row.turns, row.spend, row.saved_spend) == ("claude-auto", 3, 0.14, 0.24)
|
||||
assert row.baseline_models == {"anthropic/claude-opus-5": 3}
|
||||
assert row.baseline_model == "anthropic/claude-opus-5"
|
||||
assert row.savings_estimated_turns == 3
|
||||
assert row.savings_estimated_actual_spend == 0.14
|
||||
assert row.savings_estimated_saved_spend == 0.24
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_latest_for_key_is_none_when_the_key_wrote_no_such_session(self):
|
||||
|
|
|
|||
|
|
@ -3721,7 +3721,11 @@ class TestLLMClassifier:
|
|||
assert outcome.score is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(self, mock_router_instance):
|
||||
@pytest.mark.parametrize("redact", (False, True))
|
||||
async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(
|
||||
self, mock_router_instance: MagicMock, redact: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
|
||||
router = ComplexityRouter(
|
||||
model_name="tier-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
|
|
@ -3754,6 +3758,21 @@ class TestLLMClassifier:
|
|||
"tier-probability:complex=0.892157",
|
||||
"tier-probability:reasoning=0.980392",
|
||||
]
|
||||
redacted: Final = Router._redact_prompt_text_if_needed(
|
||||
request_kwargs={}, routing_decision=response.routing_decision
|
||||
)
|
||||
assert ("signals" in redacted) is not redact
|
||||
assert redacted["heuristic_v2_forecast"] == {
|
||||
"probabilities": {
|
||||
"SIMPLE": 11 / 102,
|
||||
"MEDIUM": 21 / 102,
|
||||
"COMPLEX": 91 / 102,
|
||||
"REASONING": 100 / 102,
|
||||
},
|
||||
"threshold": 0.8,
|
||||
"predicted_tier": "COMPLEX",
|
||||
"request_type": "general",
|
||||
}
|
||||
|
||||
def test_heuristic_v2_needs_no_classifier_model(self):
|
||||
config = ComplexityRouterConfig(classifier_type="heuristic_v2")
|
||||
|
|
@ -8879,13 +8898,29 @@ class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape:
|
|||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_decision_reaches_the_spend_log_payload(self, request_kwargs, expected_bucket):
|
||||
@pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_v2"))
|
||||
async def test_decision_reaches_the_spend_log_payload(self, request_kwargs, expected_bucket, classifier_type):
|
||||
import datetime
|
||||
import json
|
||||
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
|
||||
router = Router(model_list=self.MODEL_LIST)
|
||||
model_list: Final = [
|
||||
{
|
||||
**row,
|
||||
"litellm_params": {
|
||||
**row["litellm_params"],
|
||||
"complexity_router_config": {
|
||||
**row["litellm_params"]["complexity_router_config"],
|
||||
"classifier_type": classifier_type,
|
||||
},
|
||||
},
|
||||
}
|
||||
if row["model_name"] == "smart-router"
|
||||
else row
|
||||
for row in self.MODEL_LIST
|
||||
]
|
||||
router = Router(model_list=model_list)
|
||||
response = await router.async_pre_routing_hook(
|
||||
model="smart-router",
|
||||
request_kwargs=request_kwargs,
|
||||
|
|
@ -8915,6 +8950,15 @@ class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape:
|
|||
persisted = json.loads(payload["metadata"])["routing_decision"]
|
||||
assert persisted is not None, f"routing_decision dropped for {expected_bucket}"
|
||||
assert persisted["router_model_name"] == "smart-router"
|
||||
if classifier_type == "heuristic_v2":
|
||||
assert persisted["heuristic_v2_forecast"] == request_kwargs[expected_bucket]["routing_decision"][
|
||||
"heuristic_v2_forecast"
|
||||
]
|
||||
assert set(persisted["heuristic_v2_forecast"]["probabilities"]) == {
|
||||
"SIMPLE", "MEDIUM", "COMPLEX", "REASONING"
|
||||
}
|
||||
else:
|
||||
assert "heuristic_v2_forecast" not in persisted
|
||||
|
||||
|
||||
class TestRoutingDecisionIsPerAttempt:
|
||||
|
|
@ -9001,19 +9045,26 @@ class TestRecordRoutingDecision:
|
|||
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
assert request_kwargs == {}
|
||||
|
||||
def test_clearing_the_decision_takes_the_savings_facts_with_it(self):
|
||||
def test_clearing_the_decision_takes_the_savings_facts_with_it(self) -> None:
|
||||
"""A fallback to a plain model group re-enters the hook with the same
|
||||
`request_kwargs`. The baseline and the conversation shape ride inside the
|
||||
decision rather than beside it, so one clear cannot leave either behind and
|
||||
attribute an auto-router saving to a deployment that never routed."""
|
||||
decision = {
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
|
||||
decision: Final = {
|
||||
"router_model_name": "smart-router",
|
||||
"router_type": "complexity",
|
||||
"routed_model": "gpt-4o-mini",
|
||||
"savings_baseline_model": "anthropic/claude-opus-5",
|
||||
"savings_baseline_deployment_id": "opus-deployment",
|
||||
"conversation_continuing": False,
|
||||
}
|
||||
request_kwargs: Dict = {"litellm_metadata": {"routing_decision": decision}}
|
||||
request_kwargs: Final[dict[str, dict[str, object]]] = {"litellm_metadata": {}}
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=decision)
|
||||
stamp: Final = request_kwargs["litellm_metadata"]["_autorouter_baseline_route"]
|
||||
assert isinstance(stamp, BaselineRouteStamp)
|
||||
assert stamp.baseline_deployment_id == "opus-deployment"
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
assert request_kwargs["litellm_metadata"] == {}
|
||||
|
||||
|
|
@ -14104,6 +14155,33 @@ class TestModalityRouting:
|
|||
BASE_TIERS = {"SIMPLE": "text-cheap", "MEDIUM": "vision-mid", "COMPLEX": "vision-big"}
|
||||
BASE_VISION = {"text-cheap": False, "vision-mid": True, "vision-big": True, "vision-default": True}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_modality_escalation_preserves_the_original_heuristic_v2_forecast(
|
||||
self, mock_router_instance: MagicMock
|
||||
) -> None:
|
||||
router: Final = self._router(
|
||||
mock_router_instance,
|
||||
{
|
||||
"classifier_type": "heuristic_v2",
|
||||
"heuristic_v2_artifact": _heuristic_v2_artifact(),
|
||||
"tiers": {"COMPLEX": "text-cheap", "REASONING": "vision-big"},
|
||||
"modality_routing": True,
|
||||
},
|
||||
self.BASE_VISION,
|
||||
)
|
||||
original: Final = await router.aclassify("What color is this?")
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE
|
||||
)
|
||||
|
||||
assert original.heuristic_v2_forecast is not None
|
||||
assert result is not None and result.routing_decision is not None
|
||||
assert result.model == "vision-big"
|
||||
assert result.routing_decision["cause"] == "modality_escalation"
|
||||
assert result.routing_decision["tier"] == "REASONING"
|
||||
assert result.routing_decision["heuristic_v2_forecast"] == original.heuristic_v2_forecast
|
||||
assert result.routing_decision["heuristic_v2_forecast"]["predicted_tier"] == "COMPLEX"
|
||||
|
||||
@staticmethod
|
||||
def _router(mock_router_instance, config, vision_by_model):
|
||||
"""vision_by_model: model name -> True/False (deployment model_info) or None (undeclared)."""
|
||||
|
|
@ -14479,6 +14557,69 @@ class TestModalityRouting:
|
|||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
class TestHealthFallbackDispatch:
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("peer", (True, False), ids=("peer_failover", "default_fallback"))
|
||||
async def test_health_rewrites_preserve_the_original_heuristic_v2_forecast(self, peer: bool) -> None:
|
||||
router: Final = self._router(
|
||||
config={
|
||||
"classifier_type": "heuristic_v2",
|
||||
"heuristic_v2_artifact": _heuristic_v2_artifact(),
|
||||
"tiers": {"COMPLEX": ["primary", "peer"] if peer else "primary"},
|
||||
}
|
||||
)
|
||||
|
||||
def select_primary(models: Sequence[str]) -> str:
|
||||
return max(models)
|
||||
|
||||
with patch( # test-quality-ok: force initial classification onto the failing group in a mixed tier pool
|
||||
"litellm.router_strategy.complexity_router.complexity_router.random.choice",
|
||||
side_effect=select_primary,
|
||||
):
|
||||
original: Final = await router.async_pre_routing_hook(
|
||||
model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}]
|
||||
)
|
||||
self._unavailable(router, "primary-id", "cooldown")
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}]
|
||||
)
|
||||
|
||||
assert original is not None and original.routing_decision is not None
|
||||
assert original.model == "primary"
|
||||
assert original.routing_decision["cause"] == "heuristic_v2"
|
||||
assert result is not None and result.routing_decision is not None
|
||||
assert result.model == ("peer" if peer else "fallback")
|
||||
assert result.routing_decision["cause"] == ("health_failover" if peer else "health_default_fallback")
|
||||
assert result.routing_decision["heuristic_v2_forecast"] == original.routing_decision["heuristic_v2_forecast"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("pinned", (False, True), ids=("keyword_bypass", "session_pin"))
|
||||
async def test_heuristic_v2_bypasses_have_no_fabricated_forecast(self, pinned: bool) -> None:
|
||||
router: Final = self._router(
|
||||
session=pinned,
|
||||
config={
|
||||
"classifier_type": "heuristic_v2",
|
||||
"heuristic_v2_artifact": _heuristic_v2_artifact(),
|
||||
"tiers": {"COMPLEX": "primary"},
|
||||
"keyword_tier_rules": [{"keywords": ["quick lookup"], "tier": "COMPLEX"}],
|
||||
},
|
||||
)
|
||||
original: Final = await router.async_pre_routing_hook(
|
||||
model="health-router",
|
||||
request_kwargs={"metadata": {"session_id": "v2-forecast"}},
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
)
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="health-router",
|
||||
request_kwargs={"metadata": {"session_id": "v2-forecast"}},
|
||||
messages=[{"role": "user", "content": "quick lookup"}],
|
||||
)
|
||||
|
||||
assert original is not None and original.routing_decision is not None
|
||||
assert "heuristic_v2_forecast" in original.routing_decision
|
||||
assert result is not None and result.routing_decision is not None
|
||||
assert result.routing_decision["cause"] == ("session_affinity_pin" if pinned else "literal_keyword_match")
|
||||
assert "heuristic_v2_forecast" not in result.routing_decision
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
|
|
|||
69
tests/test_litellm/router_strategy/test_fuse_presets.py
Normal file
69
tests/test_litellm/router_strategy/test_fuse_presets.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import json
|
||||
from hashlib import sha256
|
||||
from importlib.resources import files
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets, resolve_fuse_profile
|
||||
|
||||
|
||||
def test_catalog_is_loaded_once_and_preserves_bundled_content() -> None:
|
||||
get_fuse_presets.cache_clear()
|
||||
first: Final = get_fuse_presets()
|
||||
second: Final = get_fuse_presets()
|
||||
assert first is second
|
||||
bundled: Final = json.loads(
|
||||
files("litellm.router_strategy.complexity_router").joinpath("fuse_presets.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert first.model_dump(mode="json") == bundled
|
||||
entries: Final = (*first.models, *first.harnesses)
|
||||
assert len({entry.id for entry in entries}) == len(entries)
|
||||
assert all(entry.sources and all(source.startswith("https://") for source in entry.sources) for entry in entries)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "preset_id", "expected_digest"),
|
||||
(
|
||||
("model", "gpt-6-astra-v1", "a9403b0c00ea64081b7b08b5b968850670f3a047d219a7e0668f2169146ae96e"),
|
||||
("model", "gpt-5.6-sol-v1", "2b91a6c43e0e93183aaaf9c355e1bbb8ed2e9817aab6b0c2f50148f53a23247b"),
|
||||
("model", "gpt-5.6-luna-v1", "fff94a9e01bf4519798d5be4e76a3f9d57b75a2d9966a59dc92cbfeb5cd08d07"),
|
||||
("model", "gpt-5.6-terra-v1", "75de040f3bea841fa4764885738303893ee7ac0804aed1e932cd3959185ff893"),
|
||||
("model", "claude-haiku-4-5-v1", "91c1920953073462b6b70ef810596a5325f08286b5e62630cff47938fc4157db"),
|
||||
("model", "claude-sonnet-5-v1", "133f4414c644a707cd8cf565a486153856f4836ca4e4f75ee0553f2b7a1e3663"),
|
||||
("model", "claude-opus-5-v1", "9cbfcae45d2e3a2575e44ce5adf618f56614abff4b3221d35900c647200b99ef"),
|
||||
("model", "claude-fable-5-v1", "25c275d7403f1572ffb4fe899d5feecd9a434ebdc37b4dd9ef601a8ecf4850fc"),
|
||||
("model", "claude-fable-5-1-v1", "37693107c878ab6266530395bbdc2d2813d676d179bf281d05e5a5ec1b9d4c60"),
|
||||
("harness", "unspecified-v1", "d9eb30b61509456f0c71ca805b33d821cab6605578d567a29ab421d8f602ce7b"),
|
||||
("harness", "claude-code-v1", "7ee8e9d50f1cf44a8a58461efff66d6182f245d25499702c144d1c642c101ed9"),
|
||||
("harness", "codex-cli-v1", "0678047e34562ef05b5e2fba099c1f9e5876304f7eaf3b0d8c3809e707eb3311"),
|
||||
("harness", "opencode-v1", "8b6cc240d90091ac2ef9b374b535f981a55abb91e25d4c04fdb9fc206eeb907e"),
|
||||
("harness", "mini-swe-agent-v1", "21e2dc4a8a2320a5a554a498b30516326dc3592ebf20b0f4db20ddb33e879a39"),
|
||||
),
|
||||
)
|
||||
def test_existing_preset_text_is_unchanged(
|
||||
kind: Literal["model", "harness"], preset_id: str, expected_digest: str
|
||||
) -> None:
|
||||
text: Final = resolve_fuse_profile(None, preset_id, kind)
|
||||
assert text is not None
|
||||
assert sha256(text.encode("utf-8")).hexdigest() == expected_digest
|
||||
|
||||
|
||||
def test_every_catalog_entry_resolves_without_changing_custom_ownership() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
for entry in catalog.models:
|
||||
assert resolve_fuse_profile(None, entry.id, "model") == entry.text
|
||||
assert resolve_fuse_profile("Custom text", entry.id, "model") == "Custom text"
|
||||
for entry in catalog.harnesses:
|
||||
assert resolve_fuse_profile(None, entry.id, "harness") == entry.text
|
||||
assert resolve_fuse_profile("Custom text", entry.id, "harness") == "Custom text"
|
||||
assert resolve_fuse_profile("Custom text", None, "model") == "Custom text"
|
||||
assert resolve_fuse_profile("Custom text", None, "harness") == "Custom text"
|
||||
|
||||
|
||||
def test_cached_catalog_and_records_cannot_be_modified() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
for record, field in ((catalog, "version"), (catalog.models[0], "text"), (catalog.harnesses[0], "text")):
|
||||
with pytest.raises(ValidationError, match="frozen"):
|
||||
setattr(record, field, "Changed")
|
||||
|
|
@ -11,8 +11,10 @@ from litellm import ModelResponse, Router
|
|||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.router_strategy.complexity_router.llm_v2 import (
|
||||
LLM_V2_PROMPT_VERSION,
|
||||
LLM_V2_SYSTEM_PROMPT,
|
||||
LLMV2Calibration,
|
||||
LLMV2Config,
|
||||
LLMV2ProbabilityCalibration,
|
||||
|
|
@ -174,6 +176,114 @@ def test_invalid_forecast_settings_are_rejected(overrides: dict[str, object]) ->
|
|||
LLMV2Config.model_validate({**base.model_dump(), **overrides})
|
||||
|
||||
|
||||
def _preset_config(**overrides: object) -> LLMV2Config:
|
||||
catalog: Final = get_fuse_presets()
|
||||
return LLMV2Config.model_validate(
|
||||
{
|
||||
"efficient_profile_preset": catalog.models[0].id,
|
||||
"capable_profile_preset": catalog.models[-1].id,
|
||||
"harness_preset": catalog.harnesses[-1].id,
|
||||
"max_quality_gap": 0.05,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_preset_roundtrip_keeps_references_without_materializing_text() -> None:
|
||||
config: Final = _preset_config()
|
||||
serialized: Final = config.model_dump(exclude_none=True)
|
||||
assert serialized["efficient_profile_preset"] == config.efficient_profile_preset
|
||||
assert serialized["capable_profile_preset"] == config.capable_profile_preset
|
||||
assert serialized["harness_preset"] == config.harness_preset
|
||||
assert not {"efficient_profile", "capable_profile", "harness"}.intersection(serialized)
|
||||
assert LLMV2Config.model_validate(config.model_dump()) == config
|
||||
assert LLMV2Config.model_validate_json(config.model_dump_json()) == config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_explicit_override_wins_and_survives_roundtrip(field: str) -> None:
|
||||
config: Final = _preset_config(**{field: " Operator description "})
|
||||
roundtrip: Final = LLMV2Config.model_validate_json(config.model_dump_json())
|
||||
assert roundtrip.model_dump()[field] == "Operator description"
|
||||
assert roundtrip.efficient_profile_preset == config.efficient_profile_preset
|
||||
assert roundtrip.capable_profile_preset == config.capable_profile_preset
|
||||
assert roundtrip.harness_preset == config.harness_preset
|
||||
payload: Final = json.loads(
|
||||
roundtrip.system_prompt("opaque-efficient", "opaque-capable").split("Configured solver profiles:\n")[1]
|
||||
)
|
||||
if field == "harness":
|
||||
assert payload["harness"] == "Operator description"
|
||||
else:
|
||||
assert payload[field.removesuffix("_profile")]["profile"] == "Operator description"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
@pytest.mark.parametrize("invalid", ("", " \n\t", "x" * 4001))
|
||||
def test_preset_does_not_bypass_supplied_text_bounds(field: str, invalid: str) -> None:
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{field: invalid})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
@pytest.mark.parametrize("override", (None, "Custom override"))
|
||||
@pytest.mark.parametrize("invalid_id", ("missing-v1", ""))
|
||||
def test_preset_unknown_reference_rejects_even_when_overridden(
|
||||
field: str, override: str | None, invalid_id: str
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match=f"{field}.*preset"):
|
||||
_preset_config(**{field: override, f"{field}_preset": invalid_id})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_missing_text_and_reference_rejects(field: str) -> None:
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{f"{field}_preset": None})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_reference_rejects_the_wrong_catalog_kind(field: str) -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
wrong_id: Final = catalog.models[0].id if field == "harness" else catalog.harnesses[0].id
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{f"{field}_preset": wrong_id})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ("json_schema", "json_object"))
|
||||
def test_custom_profile_prompt_bytes_are_unchanged(mode: str) -> None:
|
||||
base: Final = _config().llm_v2_config
|
||||
assert base is not None
|
||||
config: Final = LLMV2Config.model_validate({**base.model_dump(), "response_format": mode})
|
||||
old_payload: Final = {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": config.harness,
|
||||
"efficient": {"model": "opaque-efficient", "profile": config.efficient_profile},
|
||||
"capable": {"model": "opaque-capable", "profile": config.capable_profile},
|
||||
}
|
||||
schema: Final = (
|
||||
"\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema()) if mode == "json_object" else ""
|
||||
)
|
||||
assert config.system_prompt("opaque-efficient", "opaque-capable") == (
|
||||
LLM_V2_SYSTEM_PROMPT + "\n\nConfigured solver profiles:\n" + json.dumps(old_payload) + schema
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preset_router_passes_catalog_text_and_opaque_group_names_to_judge() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
config: Final = _config(llm_v2_config=_preset_config().model_dump())
|
||||
router, client = _router(_verdict().model_dump_json(), config)
|
||||
outcome: Final = await router.aclassify("Complete the supplied task")
|
||||
assert outcome.tier == ComplexityTier.SIMPLE
|
||||
prompt: Final = client.acompletion.call_args.kwargs["messages"][0]["content"]
|
||||
payload: Final = json.loads(prompt.split("Configured solver profiles:\n")[1])
|
||||
assert payload == {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": catalog.harnesses[-1].text,
|
||||
"efficient": {"model": "efficient", "profile": catalog.models[0].text},
|
||||
"capable": {"model": "capable", "profile": catalog.models[-1].text},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_judge_fuses_whole_task_and_keeps_caller_text_out_of_system_prompt() -> None:
|
||||
router, client = _router(_verdict().model_dump_json())
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue