diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 94152aeb7a0..f1456b1fde8 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -419,6 +419,7 @@ jobs: - shard: router test-path: >- + tests/unit/router_rust tests/unit/test_router python-version: "3.12" workers: 4 diff --git a/cookbook/skill_scanner/README.md b/cookbook/skill_scanner/README.md new file mode 100644 index 00000000000..2a68e116002 --- /dev/null +++ b/cookbook/skill_scanner/README.md @@ -0,0 +1,60 @@ +# Manual skill security scans + +This standalone adapter runs NVIDIA SkillSpector on a separate machine. It is not installed in the LiteLLM proxy and does not require a shared Helm or Compose deployment + +Start it with Python 3.12 or newer and uv. The script pins SkillSpector and its HTTP dependencies + +```sh +export SKILL_SCANNER_API_KEY="$(openssl rand -hex 32)" +uv run server.py +``` + +For another host, place the loopback service behind an HTTPS reverse proxy that forwards the Authorization header and enforces a 65 MiB request limit. Run it under an unprivileged account with a writable temporary directory and no production files or credentials. The scanner host needs Git and network access to requested sources and optional model providers. Apply network egress restrictions suitable for those sources + +Configure the same scanner secret in the gateway's environment, then add this to LiteLLM's configuration + +```yaml +general_settings: + skill_scanner: + endpoint: https://scanner.example.com/scan + api_key_env: SKILL_SCANNER_API_KEY + use_llm: false + timeout_seconds: 180 +``` + +The endpoint must be reachable from the gateway. A customer's laptop works while it is reachable through their network, VPN or authenticated HTTPS tunnel. `localhost` refers to the gateway machine, not the admin's browser. HTTP is accepted only for loopback development + +Open Skills, Submissions, and Review as a proxy admin. Click **Scan for threats**, inspect the report, then independently approve or reject the proposal. Scanning is optional and never changes approval permissions or blocks publication. The report shows findings, locations, remediation, risk score, scanner version and whether semantic analysis actually ran. Incomplete reports do not establish safety + +Uploaded ZIPs are sent privately to the scanner with their exact stored bytes. Metadata-only edits reuse the matching stored archive. If that archive is missing or changed, submit a new ZIP before scanning. The displayed SHA-256 identifies that archive, and the scan examines its files rather than proposed catalog metadata. Remote URL reports describe the source at scan time and do not freeze its contents. GitHub branch/tag refs and explicit-ref subdirectories are supported. Other ref/subdirectory combinations require a ZIP so the adapter does not silently scan the wrong revision + +The latest scan result is retained with the submission. Running scans have a deadline. If a gateway restarts or the scanner becomes unavailable, the attempt expires and an admin can retry. Retries replace the latest report. Approving or rejecting during a scan does not cancel it or change the decision + +Set `use_llm: true` to request semantic analysis and configure the provider on the scanner machine using [SkillSpector's provider instructions](https://github.com/NVIDIA/skillspector#llm-analysis). Provider credentials stay on that machine. Static analysis can also call OSV for dependency checks + +## HTTP contract + +The gateway sends an authenticated `POST` to the configured endpoint. The body contains form fields `source` (JSON source descriptor), `use_llm` (boolean), and `expires_at` (timezone-aware ISO timestamp), plus `file` (ZIP) when bytes are available. The service returns HTTP 200 with this JSON shape + +```json +{ + "scanner_version": "2.12.0", + "complete": true, + "scan_mode": "static-only", + "risk_score": 70, + "findings": [{ + "rule_id": "example-rule", + "severity": "HIGH", + "message": "Potential data exfiltration", + "file": "SKILL.md", + "start_line": 8, + "explanation": "Instructions request sending local files to an external host", + "remediation": "Remove the external upload instruction" + }], + "limitations": [] +} +``` + +Reports are limited to 1 MiB and 500 findings. Scanner uploads allow 64 MiB because the gateway can expand compressed skill uploads into stored compatibility ZIPs. The gateway rejects malformed reports, refuses redirects and verifies TLS. Only the configured scanner credential is sent. A timeout, HTTP error or incomplete report remains visible to the reviewer + +The bundled adapter runs at most two child scans concurrently. Each child and its descendants are terminated at the gateway deadline, capped at 570 seconds. Keep the gateway and scanner clocks synchronized. Direct adapter clients may omit `expires_at` to use the cap. Child processes never execute submitted skill scripts. Raise the gateway deadline up to 600 seconds when needed for semantic analysis and configure the reverse proxy timeout to match diff --git a/cookbook/skill_scanner/server.py b/cookbook/skill_scanner/server.py new file mode 100644 index 00000000000..44a1cecb00f --- /dev/null +++ b/cookbook/skill_scanner/server.py @@ -0,0 +1,172 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "fastapi==0.162.0", "uvicorn==0.54.0", "python-multipart==0.0.33", +# "skillspector @ git+https://github.com/NVIDIA/skillspector.git@62c2fe8622f580ac292d4078c3b71f790194dbbd", +# ] +# /// +import asyncio +import hmac +import json +import os +import signal +import sys +import tempfile +from contextlib import suppress +from datetime import datetime, timezone +from pathlib import Path +from typing import Annotated, Final +from urllib.parse import quote, urlsplit + +from fastapi import Depends, FastAPI, File, Form, HTTPException, UploadFile +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from pydantic import AwareDatetime, TypeAdapter, ValidationError + +MAX_UPLOAD: Final = 64 * 1024 * 1024 +MAX_REPORT: Final = 1024 * 1024 +SCAN_TIMEOUT: Final = 570 +SOURCE: Final = TypeAdapter(dict[str, str]) +SLOTS: Final = asyncio.Semaphore(2) +app: Final = FastAPI(title="SkillSpector adapter", docs_url=None, redoc_url=None, openapi_url=None) + + +def authenticate(credentials: Annotated[HTTPAuthorizationCredentials, Depends(HTTPBearer())]) -> None: + token: Final = os.environ.get("SKILL_SCANNER_API_KEY", "") + if not token or not hmac.compare_digest(credentials.credentials, token): + raise HTTPException(401, "Invalid scanner credentials") + + +def source_target(source: dict[str, str]) -> str: + target: Final = ( + "https://github.com/" + source.get("repo", "") if source.get("source") == "github" else source.get("url", "") + ) + parsed: Final = urlsplit(target) + if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.fragment: + raise HTTPException(400, "Remote sources require an HTTPS URL without credentials or a fragment") + ref: Final = source.get("ref") + subdir: Final = source.get("path") + if ref or subdir: + if parsed.hostname != "github.com" or len(parsed.path.strip("/").split("/")) != 2 or not ref: + raise HTTPException(400, "Upload a ZIP to scan this source's exact ref and subdirectory") + suffix: Final = "/" + quote(subdir, safe="/") if subdir else "" + return target.removesuffix(".git").rstrip("/") + "/tree/" + quote(ref, safe="/") + suffix + return target + + +async def worker(target: str, output: Path, use_llm: bool) -> None: + from skillspector.mcp_server import run_scan + + result: Final = await run_scan(target, use_llm=use_llm, allow_local_targets=not target.startswith("https://")) + coverage: Final = result.get("analysis_completeness", {}) + findings: Final = result["findings"] + complete: Final = ( + result.get("execution_successful") is True + and coverage.get("is_complete") is True + and (not use_llm or (result.get("llm_available") is True and result.get("llm_used") is True)) + and len(findings) <= 500 + ) + limitations: Final = tuple(str(value)[:2000] for value in coverage.get("limitations", ())) + report: Final = { + "scanner_version": result["version"], + "complete": complete, + "scan_mode": result["scan_mode"], + "risk_score": result["risk_score"], + "findings": [ + { + "rule_id": finding["id"], + "severity": finding["severity"], + "message": finding.get("pattern") or finding.get("finding") or "Security finding", + "file": finding["location"]["file"], + "start_line": finding["location"].get("start_line"), + "explanation": finding.get("explanation"), + "remediation": finding.get("remediation"), + } + for finding in findings[:500] + ], + "limitations": ( + *limitations, + *(("Only the first 500 findings are displayed.",) if len(findings) > 500 else ()), + ), + } + encoded: Final = json.dumps(report).encode() + if len(encoded) > MAX_REPORT: + raise ValueError("Scanner report exceeded the response limit") + output.write_bytes(encoded) + + +async def run_worker(target: str, output: Path, use_llm: bool, expires_at: datetime | None = None) -> dict[str, object]: + timeout: Final = ( + SCAN_TIMEOUT + if expires_at is None + else min(SCAN_TIMEOUT, (expires_at - datetime.now(timezone.utc)).total_seconds()) + ) + if timeout <= 0: + raise HTTPException(504, "SkillSpector scan deadline expired") + process: Final = await asyncio.create_subprocess_exec( + sys.executable, + str(Path(__file__).resolve()), + "--scan", + target, + str(output), + str(use_llm).lower(), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + env={ + **os.environ, + "HOME": str(output.parent), + "TMPDIR": str(output.parent), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ALLOW_PROTOCOL": "https", + }, + ) + try: + code: Final = await asyncio.wait_for(process.wait(), timeout) + if code != 0 or not output.is_file() or output.stat().st_size > MAX_REPORT: + raise HTTPException(502, "SkillSpector could not produce a report for this source") + return TypeAdapter(dict[str, object]).validate_json(output.read_bytes()) + except TimeoutError: + raise HTTPException(504, "SkillSpector scan timed out") from None + finally: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + await process.wait() + + +@app.post("/scan", dependencies=[Depends(authenticate)]) +async def scan( + source: Annotated[str, Form(max_length=16384)], + use_llm: Annotated[bool, Form()] = False, + expires_at: Annotated[AwareDatetime | None, Form()] = None, + file: Annotated[UploadFile | None, File()] = None, +) -> dict[str, object]: + if SLOTS.locked(): + raise HTTPException(429, "Scanner is busy; try again shortly") + try: + parsed: Final = SOURCE.validate_json(source) + except ValidationError: + raise HTTPException(400, "Invalid source metadata") from None + async with SLOTS: + with tempfile.TemporaryDirectory(prefix="skill-scan-") as directory: + root: Final = Path(directory) + if file is not None: + content: Final = await file.read(MAX_UPLOAD + 1) + if len(content) > MAX_UPLOAD: + raise HTTPException(413, "Upload exceeds 64 MiB") + archive: Final = root / "skill.zip" + archive.write_bytes(content) + return await run_worker(str(archive), root / "report.json", use_llm, expires_at) + return await run_worker(source_target(parsed), root / "report.json", use_llm, expires_at) + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "--scan": + asyncio.run(worker(sys.argv[2], Path(sys.argv[3]), sys.argv[4] == "true")) + else: + import uvicorn + + if not os.environ.get("SKILL_SCANNER_API_KEY"): + raise SystemExit("Set SKILL_SCANNER_API_KEY before starting the scanner") + uvicorn.run(app, host="127.0.0.1", port=int(os.environ.get("PORT", "8099"))) diff --git a/docs/ldap_sync.md b/docs/ldap_sync.md index 072c59aeaa2..a2ebcea0b7b 100644 --- a/docs/ldap_sync.md +++ b/docs/ldap_sync.md @@ -50,8 +50,8 @@ LDAP is authoritative for the membership of every matched team, including manual A group no longer returned by LDAP leaves its team and memberships unchanged. A successfully read group with no members removes all memberships from its corresponding team. No team or user is ever deleted, and no user is provisioned by this sync. -Failed or incomplete group searches, referrals, duplicate group IDs, invalid responses and LDAP operation exceptions abort the directory snapshot without applying it. A member DN whose entry no longer exists skips only its group, so other groups still synchronize and that team keeps its current members until the DN is removed or restored. Unreadable entries still abort the snapshot. Other member validation failures, such as ambiguous identities or nested groups, also skip their affected group. A membership-write failure for one valid group does not prevent other valid groups from synchronizing. A user disappearing during the membership write causes that group's transaction to make no changes. Repeated successful runs with the same roster do not rewrite it. +Failed or incomplete group searches, referrals, duplicate group IDs, invalid responses and LDAP operation exceptions abort the directory snapshot without applying it. A member DN whose entry no longer exists skips only its group, so other groups still synchronize and that team keeps its current members until the DN is removed or restored. Unreadable entries still abort the snapshot. Groups with missing, blank or multivalued IDs skip individually. Other member validation failures, such as ambiguous identities or nested groups, also skip their affected group. A membership-write failure for one valid group does not prevent other valid groups from synchronizing. A user disappearing during the membership write causes that group's transaction to make no changes. Repeated successful runs with the same roster do not rewrite it. The LDAP identity must have permission to read every selected group, its complete membership attributes, and the user identity attributes. LDAP can omit attributes hidden by access controls just as it omits empty attributes, so the sync cannot distinguish every permission restriction from an empty group. Verify the bind account and dry-run output before enabling writes. -Runs have a timeout shorter than their shared Redis lease. A timeout cancels further writes and signals the LDAP reader to stop between entries and lookups. LDAP socket timeouts and server search time limits use `timeout_seconds` as an upper bound and are capped further for short sync intervals, and the next interval retries from current state. Cancellation does not undo groups already committed earlier in the run. Stopping the sync (`enabled: false` or removing the block and restarting) leaves current teams and memberships in place. +Runs have a timeout shorter than their shared Redis lease. A timeout cancels further writes and signals the LDAP reader to stop between entries and lookups. LDAP socket timeouts and server search time limits use `timeout_seconds` as an upper bound and are capped further for short sync intervals, and the next interval retries from current state. Cancellation does not undo groups already committed earlier in the run. Cache cleanup for a committed membership change finishes before cancellation is reported, so removed keys are not left cached by the job timeout. This cleanup can extend past the run deadline but performs no further membership writes. Stopping the sync (`enabled: false` or removing the block and restarting) leaves current teams and memberships in place. diff --git a/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py b/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py index a2322d7f650..bb3b712d50b 100644 --- a/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py +++ b/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py @@ -1,9 +1,9 @@ import enum -from typing import Dict, List, Optional from pydantic import Field from litellm.proxy._types import WebhookEvent +from litellm.types.integrations.model_deprecation_email import MODEL_DEPRECATION_EMAIL_EVENT from litellm.types.llms.base import LiteLLMBaseModel @@ -26,7 +26,7 @@ class SendKeyCreatedEmailEvent(WebhookEvent): class SendKeyRotatedEmailEvent(WebhookEvent): virtual_key: str - key_alias: Optional[str] = None + key_alias: str | None = None """ The virtual key that was rotated this will be sk-123xxx, since we will be emailing this to the user to start using the new key @@ -34,23 +34,33 @@ class SendKeyRotatedEmailEvent(WebhookEvent): class EmailEvent(str, enum.Enum): + model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT virtual_key_created = "Virtual Key Created" new_user_invitation = "New User Invitation" virtual_key_rotated = "Virtual Key Rotated" soft_budget_crossed = "Soft Budget Crossed" max_budget_alert = "Max Budget Alert" + class EmailEventSettings(LiteLLMBaseModel): event: EmailEvent enabled: bool + + class EmailEventSettingsUpdateRequest(LiteLLMBaseModel): - settings: List[EmailEventSettings] + settings: list[EmailEventSettings] + + class EmailEventSettingsResponse(LiteLLMBaseModel): - settings: List[EmailEventSettings] + settings: list[EmailEventSettings] + + class DefaultEmailSettings(LiteLLMBaseModel): """Default settings for email events""" - settings: Dict[EmailEvent, bool] = Field( + + settings: dict[EmailEvent, bool] = Field( default_factory=lambda: { + EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders. EmailEvent.virtual_key_created: True, # On by default EmailEvent.new_user_invitation: True, # On by default EmailEvent.virtual_key_rotated: True, # On by default @@ -58,10 +68,12 @@ class DefaultEmailSettings(LiteLLMBaseModel): EmailEvent.max_budget_alert: True, # On by default } ) - def to_dict(self) -> Dict[str, bool]: + + def to_dict(self) -> dict[str, bool]: """Convert to dictionary with string keys for storage""" return {event.value: enabled for event, enabled in self.settings.items()} + @classmethod - def get_defaults(cls) -> Dict[str, bool]: + def get_defaults(cls) -> dict[str, bool]: """Get the default settings as a dictionary with string keys""" - return cls().to_dict() \ No newline at end of file + return cls().to_dict() diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql new file mode 100644 index 00000000000..b00cf892b68 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql @@ -0,0 +1,19 @@ +SET LOCAL lock_timeout = '10s'; +CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" ( + "delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + "request_id" TEXT NOT NULL, + "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending', + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "applied_at" TIMESTAMP(3), + "last_error" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "accounting_payload" JSONB NOT NULL, + "is_batch" BOOLEAN NOT NULL +); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_next_attempt_at_idx" + ON "LiteLLM_SpendLogDelivery" ("state", "next_attempt_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_applied_at_idx" + ON "LiteLLM_SpendLogDelivery" ("state", "applied_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_request_id_idx" + ON "LiteLLM_SpendLogDelivery" ("request_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql new file mode 100644 index 00000000000..cba6a18368f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql @@ -0,0 +1,5 @@ +SET LOCAL lock_timeout = '10s'; +DROP TABLE IF EXISTS "LiteLLM_AccountingDelivery"; +DROP TABLE IF EXISTS "LiteLLM_AccountingEvent"; +DROP TYPE IF EXISTS "AccountingDeliveryTarget"; +DROP TYPE IF EXISTS "AccountingEventKind"; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_skill_version_history/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_skill_version_history/migration.sql new file mode 100644 index 00000000000..ecfa7321d87 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_skill_version_history/migration.sql @@ -0,0 +1,16 @@ +SET LOCAL lock_timeout = '10s'; +CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginVersion" ( + "id" TEXT NOT NULL, + "plugin_id" TEXT NOT NULL, + "version" TEXT, + "description" TEXT, + "manifest_json" TEXT, + "published_at" TIMESTAMP(3), + "published_by" TEXT, + "archive_available" BOOLEAN NOT NULL DEFAULT false, + "content" BYTEA, + CONSTRAINT "LiteLLM_ClaudeCodePluginVersion_pkey" PRIMARY KEY ("id"), + CONSTRAINT "LiteLLM_ClaudeCodePluginVersion_plugin_id_fkey" FOREIGN KEY ("plugin_id") REFERENCES "LiteLLM_ClaudeCodePluginTable"("id") ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ClaudeCodePluginVersion_plugin_id_version_key" ON "LiteLLM_ClaudeCodePluginVersion"("plugin_id", "version"); +CREATE INDEX IF NOT EXISTS "LiteLLM_ClaudeCodePluginVersion_plugin_id_published_at_idx" ON "LiteLLM_ClaudeCodePluginVersion"("plugin_id", "published_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105010000_skill_submissions/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105010000_skill_submissions/migration.sql new file mode 100644 index 00000000000..e7975258ca9 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105010000_skill_submissions/migration.sql @@ -0,0 +1,27 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_SkillSubmission" ( + "id" TEXT NOT NULL, + "plugin_name" TEXT NOT NULL, + "operation" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'pending_review', + "proposal_json" TEXT NOT NULL, + "archive_content" BYTEA, + "submitted_by" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "base_plugin_id" TEXT, + "base_updated_at" TIMESTAMP(3), + "reviewed_by" TEXT, + "reviewed_at" TIMESTAMP(3), + "rejection_reason" TEXT, + CONSTRAINT "LiteLLM_SkillSubmission_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_submitted_by_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("submitted_by", "created_at" DESC, "id" DESC); +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_submitted_by_status_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("submitted_by", "status", "created_at" DESC, "id" DESC); +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_status_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("status", "created_at" DESC, "id" DESC); +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("created_at" DESC, "id" DESC); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105020000_skill_scanning/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105020000_skill_scanning/migration.sql new file mode 100644 index 00000000000..6b70cca4a9a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105020000_skill_scanning/migration.sql @@ -0,0 +1,3 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_SkillSubmission" ADD COLUMN IF NOT EXISTS "scan_json" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql new file mode 100644 index 00000000000..43e26b324a0 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql @@ -0,0 +1,37 @@ +SET LOCAL lock_timeout = '10s'; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = to_regclass('"LiteLLM_ClaudeCodePluginTable"') + AND attname = 'anonymous_download_count' AND NOT attisdropped + ) THEN + ALTER TABLE "LiteLLM_ClaudeCodePluginTable" + ADD COLUMN IF NOT EXISTS "anonymous_download_count" BIGINT NOT NULL DEFAULT 0; + END IF; + + IF to_regclass('"LiteLLM_ClaudeCodePluginDownload"') IS NULL THEN + CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginDownload" ( + "id" BIGSERIAL NOT NULL, + "plugin_id" TEXT NOT NULL, + "downloaded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "version" TEXT, + "digest" TEXT NOT NULL, + "authenticated" BOOLEAN NOT NULL, + "user_id" TEXT, + "user_email" TEXT, + "key_alias" TEXT, + "team_id" TEXT, + "team_alias" TEXT, + CONSTRAINT "LiteLLM_ClaudeCodePluginDownload_pkey" PRIMARY KEY ("id"), + CONSTRAINT "LiteLLM_ClaudeCodePluginDownload_plugin_id_fkey" FOREIGN KEY ("plugin_id") + REFERENCES "LiteLLM_ClaudeCodePluginTable"("id") ON DELETE CASCADE ON UPDATE CASCADE + ); + END IF; + + IF to_regclass('"LiteLLM_ClaudeCodePluginDownload_plugin_id_id_idx"') IS NULL THEN + CREATE INDEX IF NOT EXISTS "LiteLLM_ClaudeCodePluginDownload_plugin_id_id_idx" + ON "LiteLLM_ClaudeCodePluginDownload"("plugin_id", "id"); + END IF; +END $$; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index c007b047087..818f2dc4858 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1616,10 +1616,69 @@ model LiteLLM_ClaudeCodePluginTable { created_by String? uploaded_archive LiteLLM_ClaudeCodePluginUpload? + versions LiteLLM_ClaudeCodePluginVersion[] + downloads LiteLLM_ClaudeCodePluginDownload[] + anonymous_download_count BigInt @default(0) @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginVersion { + id String @id @default(uuid()) + plugin_id String + version String? + description String? + manifest_json String? + published_at DateTime? + published_by String? + archive_available Boolean @default(false) + content Bytes? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@unique([plugin_id, version]) + @@index([plugin_id, published_at]) +} + + +model LiteLLM_SkillSubmission { + id String @id @default(uuid()) + plugin_name String + operation String + status String @default("pending_review") + proposal_json String + archive_content Bytes? + submitted_by String + created_at DateTime @default(now()) + base_plugin_id String? + base_updated_at DateTime? + reviewed_by String? + reviewed_at DateTime? + rejection_reason String? + scan_json String? + + @@index([submitted_by, created_at(sort: Desc), id(sort: Desc)]) + @@index([submitted_by, status, created_at(sort: Desc), id(sort: Desc)]) + @@index([status, created_at(sort: Desc), id(sort: Desc)]) + @@index([created_at(sort: Desc), id(sort: Desc)]) +} + +model LiteLLM_ClaudeCodePluginDownload { + id BigInt @id @default(autoincrement()) + plugin_id String + downloaded_at DateTime @default(now()) + version String? + digest String + authenticated Boolean + user_id String? + user_email String? + key_alias String? + team_id String? + team_alias String? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@index([plugin_id, id]) +} + model LiteLLM_ClaudeCodePluginUpload { plugin_name String @id content Bytes @@ -2071,74 +2130,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ba1131d7a92..23ebfc7d3cf 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4288,6 +4288,7 @@ dependencies = [ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ + "async-trait", "aws-sdk-secretsmanager", "bytes", "criterion", @@ -4312,6 +4313,7 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-model-catalog", + "litellm-router-core", "litellm-router-types", "litellm-secret-value", "litellm-secrets", @@ -4393,6 +4395,24 @@ dependencies = [ [[package]] name = "litellm-router-core" version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "jiff", + "litellm-cache", + "litellm-router-types", + "parking_lot", + "rand 0.8.7", + "rstest", + "serde", + "serde_json", + "serde_with", + "sha2 0.10.9", + "strum", + "thiserror 2.0.19", + "tokio", + "tracing", +] [[package]] name = "litellm-router-types" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 943e8576ae6..71914a0bffe 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" } litellm-error = { path = "crates/error" } litellm-router = { path = "crates/router" } litellm-router-types = { path = "crates/router-types" } +litellm-router-core = { path = "crates/router-core" } litellm-tracing = { path = "crates/tracing" } litellm-spend-clickhouse = { path = "crates/spend-clickhouse" } litellm-storage-clickhouse = { path = "crates/storage-clickhouse" } @@ -76,6 +77,8 @@ litellm-cost = { path = "crates/cost" } litellm-model-catalog = { path = "crates/model-catalog" } litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" } +arc-swap = "1.9.2" +async-trait = "0.1.91" askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] } tracing = "0.1" axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] } diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index fd826f549e5..ba892199dcf 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -25,6 +25,7 @@ litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true litellm-storage-clickhouse.workspace = true litellm-host.workspace = true +async-trait.workspace = true bytes.workspace = true futures-util.workspace = true litellm-cache.workspace = true @@ -47,6 +48,7 @@ litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "has litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true +litellm-router-core.workspace = true litellm-host-python.workspace = true litellm-model-catalog.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index de35419308a..f4a4f7c2b26 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -12,6 +12,7 @@ mod marshal; mod model_registry; mod preflight; mod python_settings; +mod router; mod routes; mod secrets; mod tokenizer; @@ -36,6 +37,8 @@ mod _native { #[pymodule_export] use crate::model_registry::bundled_model_cost_map; #[pymodule_export] + use crate::router::{RouterCall, RouterCoreHandle}; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -115,6 +118,8 @@ mod tests { "bundled_model_cost_map", "TokenCounter", "Tokenizer", + "RouterCoreHandle", + "RouterCall", "gil_stats", "process_state_started", "reserve_process_for_forking", diff --git a/litellm-rust/crates/python-bridge/src/router/config.rs b/litellm-rust/crates/python-bridge/src/router/config.rs new file mode 100644 index 00000000000..0b46f2208c5 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/config.rs @@ -0,0 +1,408 @@ +//! The documents `RouterCoreHandle` reads: the router's static and dynamic config, a call's +//! request and an `apply` change set. A config key router-core does not read declines the +//! handle, so Python keeps the router on its own core. + +use std::collections::BTreeMap; + +use litellm_router_core::select::PassThrough; +use litellm_router_core::{ + ChangeSet, ConfigError, DeploymentSpec, OutcomeCallbacks, RouterConfig, RouterDynamicConfig, + RouterRequest, + config::{Fallbacks, RetryPolicy}, + host::UncountedRetries, + mock::MockFailure, + registry::IndexOverrides, +}; +use serde::{Deserialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +/// The `RouterConfig` keys router-core reads. +const STATIC_KEYS: &[&str] = &[ + "model_list", + "routing_strategy", + "routing_groups", + "routing_group_members", + "plugins", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_policy", + "model_group_retry_policy", + "retry_after", + "timeout", + "request_timeout", + "drop_params", + "enable_pre_call_checks", + "routing_plugins", + "callback_filters", + "enable_health_check_routing", + "health_check_staleness_threshold", + "background_health_check_model_groups", + "enable_weighted_failover", + "fallback_on_unsupported_params", + "provider_default_deployment_ids", +]; + +/// The `RouterDynamicConfig` keys router-core reads. +const DYNAMIC_KEYS: &[&str] = &[ + "version", + "model_alias_map", + "model_group_alias", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "allowed_fails_is_default", + "fallback_keys", + "default_deployment", + "default_deployment_raises", +]; + +/// Router arguments that do not change routing, accepted in either config and dropped. +const IGNORED_KEYS: &[&str] = &[ + "set_verbose", + "debug_level", + "alerting_config", + "client_ttl", +]; + +/// Fallback lists Python treats as unset when empty (`context_window_fallbacks or None`). +const EMPTY_MEANS_UNSET: &[&str] = &["context_window_fallbacks", "content_policy_fallbacks"]; + +/// Which document a failure is about. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub(crate) enum Document { + #[strum(serialize = "router config")] + Static, + #[strum(serialize = "dynamic router config")] + Dynamic, + #[strum(serialize = "router request")] + Request, + #[strum(serialize = "change set")] + Changes, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("the {document} must be a JSON object")] + NotObject { document: Document }, + #[error("router-core does not read `{key}` in the {document}")] + UnknownKey { document: Document, key: String }, + #[error("the {document} does not have the shape router-core reads")] + Shape { + document: Document, + #[source] + source: serde_json::Error, + }, + #[error("the request names no model")] + MissingModel, + #[error("router-core rejected the router config")] + Core(#[from] ConfigError), +} + +impl Error { + /// Whether router-core cannot serve this config, so Python keeps its own core. + pub(crate) fn declined(&self) -> bool { + match self { + Self::UnknownKey { .. } | Self::Core(_) => true, + Self::Shape { document, .. } => { + matches!(document, Document::Static | Document::Dynamic) + } + Self::NotObject { .. } | Self::MissingModel => false, + } + } +} + +fn object(document: Document, value: Value) -> Result, Error> { + match value { + Value::Object(object) => Ok(object), + _ => Err(Error::NotObject { document }), + } +} + +fn shaped(document: Document, object: Map) -> Result { + serde_json::from_value(Value::Object(object)) + .map_err(|source| Error::Shape { document, source }) +} + +/// Drop unset and ignored keys, then reject any key outside `known`. +fn routing_keys( + document: Document, + value: Value, + known: &[&str], +) -> Result, Error> { + let mut object = object(document, value)?; + object.retain(|key, value| { + let empty_unset = EMPTY_MEANS_UNSET.contains(&key.as_str()) + && value.as_array().is_some_and(Vec::is_empty); + !(value.is_null() || empty_unset || IGNORED_KEYS.contains(&key.as_str())) + }); + if let Some(key) = object.keys().find(|key| !known.contains(&key.as_str())) { + return Err(Error::UnknownKey { + document, + key: key.clone(), + }); + } + Ok(object) +} + +pub(crate) fn parse_router_config(value: Value) -> Result { + let object = routing_keys(Document::Static, value, STATIC_KEYS)?; + shaped(Document::Static, object) +} + +pub(crate) fn parse_dynamic_config(value: Value) -> Result { + let object = routing_keys(Document::Dynamic, value, DYNAMIC_KEYS)?; + shaped(Document::Dynamic, object) +} + +/// A call's request as Python spells it. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestSpec { + model: Option, + team_id: Option, + listing_team_id: Option, + proxy_admin: Option, + allowed_models: Option>, + admin_model: Option, + specific_deployment: Option, + num_retries: Option, + uncounted_retries: Option, + disable_fallbacks: Option, + router_weights: Option, + pinned_deployment_id: Option, + routing_strategy: Option, + filter_tags: Option, + /// `listed` or `healthy`: which pass-through pick the request makes. + pass_through: Option, + target_order: Option, + excluded_ids: Option>, + retry_skipped_ids: Option>, + pre_routes: Option, + has_image: Option, + endpoint: Option, + web_search: Option, + route_handler: Option, + decisions_media: Option, + wildcard_request: Option, + unrouted: Option, + passthrough_on_no_deployment: Option, + skip_weighted_failover: Option, + same_group_fallbacks_only: Option, + model_group_retry_policy: Option>, + retry_policy_raises: Option>, + #[serde(default, deserialize_with = "present")] + fallbacks: Option>, + #[serde(default, deserialize_with = "present")] + context_window_fallbacks: Option>, + #[serde(default, deserialize_with = "present")] + content_policy_fallbacks: Option>, + max_fallbacks: Option, + mock_failure: Option, + cooldown_time: Option, + health_check_probe: Option, + outcome_callbacks: Option, + body: Option, +} + +/// A key the request passed, even as `None`: `Some(None)`. +fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) +} + +/// Read a request; `model` replaces the request's own when given. +pub(crate) fn parse_request(value: Value, model: Option<&str>) -> Result { + let spec: RequestSpec = shaped(Document::Request, object(Document::Request, value)?)?; + let model = model + .map(str::to_owned) + .or(spec.model) + .ok_or(Error::MissingModel)?; + Ok(RouterRequest { + model, + body: spec.body.unwrap_or_default(), + team_id: spec.team_id, + listing_team_id: spec.listing_team_id, + proxy_admin: spec.proxy_admin.unwrap_or(false), + allowed_models: spec.allowed_models.unwrap_or_default(), + admin_model: spec.admin_model, + specific_deployment: spec.specific_deployment.unwrap_or(false), + num_retries: spec.num_retries, + uncounted_retries: spec.uncounted_retries, + disable_fallbacks: spec.disable_fallbacks.unwrap_or(false), + router_weights: spec.router_weights, + pinned_deployment_id: spec.pinned_deployment_id, + routing_strategy: spec.routing_strategy, + filter_tags: spec.filter_tags.unwrap_or(false), + pass_through: match spec.pass_through.as_deref() { + None => None, + Some("healthy") => Some(PassThrough::Healthy), + Some(_) => Some(PassThrough::Listed), + }, + target_order: spec.target_order, + excluded_ids: spec.excluded_ids.unwrap_or_default(), + retry_skipped_ids: spec.retry_skipped_ids.unwrap_or_default(), + pre_routes: spec.pre_routes.unwrap_or(false), + has_image: spec.has_image.unwrap_or(false), + endpoint: spec.endpoint, + web_search: spec.web_search.unwrap_or(false), + route_handler: spec.route_handler, + decisions_media: spec.decisions_media.unwrap_or(false), + wildcard_request: spec.wildcard_request, + unrouted: spec.unrouted.unwrap_or(false), + passthrough_on_no_deployment: spec.passthrough_on_no_deployment.unwrap_or(false), + skip_weighted_failover: spec.skip_weighted_failover.unwrap_or(false), + same_group_fallbacks_only: spec.same_group_fallbacks_only.unwrap_or(false), + model_group_retry_policy: spec.model_group_retry_policy, + retry_policy_raises: spec.retry_policy_raises.unwrap_or_default(), + fallbacks: spec.fallbacks, + context_window_fallbacks: spec.context_window_fallbacks, + content_policy_fallbacks: spec.content_policy_fallbacks, + max_fallbacks: spec.max_fallbacks, + mock_failure: spec.mock_failure, + cooldown_time: spec.cooldown_time, + health_check_probe: spec.health_check_probe.unwrap_or(false), + outcome_callbacks: spec.outcome_callbacks.unwrap_or_default(), + payload: None, + }) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ChangesSpec { + /// Every deployment, replacing the current list before the upserts and deletes. + #[serde(default)] + replace: Option>, + /// Deployments added at the end, beside any that share their id. + #[serde(default)] + append: Vec, + #[serde(default)] + upsert: Vec, + #[serde(default)] + delete: Vec, + /// A new dynamic document (the same keys `RouterCoreHandle(static, dynamic)` takes). + #[serde(default)] + settings: Option, + /// The registry indexes the host wrote (`IndexOverrides`); `{}` drops them. + #[serde(default)] + index: Option, + /// The deployment at the id map's position for this id, removed with every index shifted. + #[serde(default)] + remove: Option, + /// A write to the indexes the host hands out (`HostIndexes`), field by field. + #[serde(default)] + host_index: Option, +} + +pub(crate) fn parse_changes(value: Value) -> Result { + let spec: ChangesSpec = shaped(Document::Changes, object(Document::Changes, value)?)?; + Ok(ChangeSet { + replace: spec.replace, + appends: spec.append, + upserts: spec.upsert, + deletes: spec.delete, + settings: spec.settings.map(parse_dynamic_config).transpose()?, + index: spec.index, + remove: spec.remove, + host_index: spec.host_index, + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::unknown_key(json!({"routing_strategy_args": {"ttl": 1}}), "routing_strategy_args")] + #[case::dynamic_key_in_static(json!({"num_retries": 2}), "num_retries")] + fn an_unread_static_key_declines_naming_the_key(#[case] json: Value, #[case] key: &str) { + let error = parse_router_config(json).unwrap_err(); + assert!(error.declined()); + assert!(matches!(&error, Error::UnknownKey { key: named, .. } if named == key)); + } + + #[rstest] + #[case::null_unknown(json!({"routing_strategy_args": null}))] + #[case::ignored(json!({"set_verbose": true, "debug_level": "INFO"}))] + #[case::pre_call_checks_off(json!({"enable_pre_call_checks": false}))] + #[case::pre_call_checks_on(json!({"enable_pre_call_checks": true}))] + fn unset_and_ignored_keys_are_accepted(#[case] json: Value) { + parse_router_config(json).unwrap(); + } + + #[rstest] + #[case::not_object(json!([]), false)] + #[case::bad_shape(json!({"retry_after": "soon"}), true)] + fn a_config_that_is_not_an_object_is_a_value_error_and_a_wrong_shape_declines( + #[case] json: Value, + #[case] declined: bool, + ) { + assert_eq!(parse_router_config(json).unwrap_err().declined(), declined); + } + + #[rstest] + fn python_router_defaults_read_as_unset() { + let dynamic = parse_dynamic_config(json!({ + "num_retries": null, "fallbacks": [], "context_window_fallbacks": [], + "content_policy_fallbacks": [], "model_group_alias": {"gpt": "gpt-4"}, + })) + .unwrap(); + assert_eq!(dynamic.num_retries, None); + assert!(dynamic.context_window_fallbacks.is_none()); + assert!(dynamic.content_policy_fallbacks.is_none()); + assert_eq!(dynamic.model_alias_map["gpt"], "gpt-4"); + } + + #[rstest] + #[case::own_model(json!({"model": "gpt", "num_retries": 1}), None, "gpt")] + #[case::override_model(json!({"model": "gpt"}), Some("other"), "other")] + fn a_request_reads_its_model( + #[case] json: Value, + #[case] model: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!(parse_request(json, model).unwrap().model, expected); + } + + #[rstest] + #[case::negative("negative", UncountedRetries::Negative)] + #[case::not_positive("not_positive", UncountedRetries::NotPositive)] + #[case::raises("raises", UncountedRetries::Raises)] + fn a_request_reads_a_num_retries_that_is_no_count( + #[case] spelled: &str, + #[case] expected: UncountedRetries, + ) { + let request = + parse_request(json!({"model": "gpt", "uncounted_retries": spelled}), None).unwrap(); + assert_eq!(request.uncounted_retries, Some(expected)); + assert_eq!(request.num_retries, None); + } + + #[rstest] + #[case::unknown_field(json!({"model": "gpt", "temperature": 1}))] + #[case::no_model(json!({"team_id": "t"}))] + fn a_bad_request_is_a_value_error(#[case] json: Value) { + assert!(!parse_request(json, None).unwrap_err().declined()); + } + + #[rstest] + fn a_change_set_reads_upserts_and_deletes() { + let changes = parse_changes(json!({ + "upsert": [{"model_name": "m", "litellm_params": {"model": "openai/x"}}], + "delete": ["old"], + })) + .unwrap(); + assert_eq!(changes.upserts.len(), 1); + assert_eq!(changes.deletes, ["old"]); + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/host.rs b/litellm-rust/crates/python-bridge/src/router/host.rs new file mode 100644 index 00000000000..16b425d69c6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/host.rs @@ -0,0 +1,1569 @@ +//! The host behind `RouterCoreHandle`. Rust never awaits Python: each host operation the call +//! loop needs becomes a [`HostCall`] parked in the call's [`Exchange`], and the call future stays +//! pending until Python runs it and hands the [`HostCallResponse`] back through [`Driver::resume`]. +//! +//! The driver polls the call future with a no-op waker while the call's exchange is bound to the +//! current thread, so a host future finds its exchange without the router passing it through. +//! +//! The routing cache is a host operation too: [`StepCache`] parks every [`RouterCache`] op as a +//! `cache` step, and Python runs it against the router's own `DualCache` (memory plus Redis), so +//! Python and Rust read and write the same cooldowns, fail counters and usage. + +use std::{ + cell::RefCell, + future::{Future, poll_fn}, + pin::Pin, + sync::{Arc, Mutex, MutexGuard, PoisonError}, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use async_trait::async_trait; +use litellm_router_core::{ + Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, Deployment, Error, Host, Output, + Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache, RouterEvent, RouterRequest, + RouterResponse, RoutingContext, SecretRef, SelectStage, ThreadRandomness, + build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo, + precall::PreCallFacts, +}; +use pyo3::{PyResult, Python, types::PyAnyMethods}; +use serde_json::{Map, Value}; + +/// The `kind` of a step dict. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] +pub(crate) enum StepKind { + #[strum(serialize = "attempt")] + Attempt, + #[strum(serialize = "sleep")] + Sleep, + #[strum(serialize = "allow_fallback")] + AllowFallback, + #[strum(serialize = "refuse_fallbacks")] + RefuseFallbacks, + #[strum(serialize = "select")] + Select, + #[strum(serialize = "pre_route")] + PreRoute, + #[strum(serialize = "cache")] + Cache, + #[strum(serialize = "secrets")] + Secrets, + #[strum(serialize = "validate")] + Validate, + #[strum(serialize = "provider")] + Provider, + #[strum(serialize = "added")] + Added, + #[strum(serialize = "model_info")] + ModelInfo, + #[strum(serialize = "done")] + Done, + #[strum(serialize = "failed")] + Failed, +} + +/// Which selection stage Python is asked to run. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Stage { + /// Pre-call checks and sub-routers: return the deployments that remain. + Plugins, + /// A strategy router-core does not implement: return the candidates with the pick first. + Strategy(String), + /// Tag routing: return the deployments that remain. + Tags, + /// `enable_pre_call_checks`: return the facts the filter decides by. + PreCall, + /// `_deployment_declares_no_vision`: return the candidates that declare no vision support. + TextOnly, + /// `_deployment_serves_route`: return the candidates that cannot serve `handler`'s call. + RouteIncapable { handler: String }, + /// `_decisions_deployment_refuses_media`: return the candidates that refuse the call's media. + MediaRefusing, + /// The routing plugins: run them when `run`, then return the candidate models they left. + RoutingPlugins { run: bool }, + /// The host's `CustomLogger` deployment filters: return the ids that remain, or a failure. + /// They read the ids this request already failed on, as Python's request kwargs carry them. + CallbackFilters { + excluded_ids: Vec, + skipped_ids: Vec, + target_order: Option, + }, + /// `f"{provider}/{model}"` for a name no wildcard pattern matches as spelled: return it as + /// the one entry of `models`. + ProviderRequest, +} + +/// The `op` of a `cache` step: one per [`RouterCache`] method. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] +pub(crate) enum CacheOp { + /// `async_get_cache(key)`: answered with the stored JSON value, or `None`. + #[strum(serialize = "get")] + Get, + /// `async_set_cache(key, value, ttl)`: answered with `None`. + #[strum(serialize = "set")] + Set, + /// `async_increment(key, value, ttl)`: answered with the new number. The TTL applies when + /// the key is created. + #[strum(serialize = "increment")] + Increment, + /// `async_increment_usage(keys, value, ttl)`: `value` is one amount per key, added as + /// `_increment_deployment_usage` adds them. Answered with `None`. + #[strum(serialize = "increment_usage")] + IncrementUsage, + /// `async_batch_get_cache(keys)`: answered with one JSON value or `None` per key, in order. + #[strum(serialize = "batch_get")] + BatchGet, + /// `async_delete_cache(key)`: answered with `None`. + #[strum(serialize = "delete")] + Delete, +} + +/// Python's outcome of a `cache` step. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum CacheOutcome { + /// `get`: the stored value (`None` when missing). + Value(Option), + /// `batch_get`: one entry per key. + Values(Vec>), + /// `increment`: the counter after the increment. + Number(f64), + /// `set`, `delete` and `increment_usage`. + Stored, +} + +impl CacheOutcome { + /// The op this outcome answers (`Stored` answers both `set` and `delete`). + fn answers(&self, op: CacheOp) -> bool { + matches!( + (self, op), + (Self::Value(_), CacheOp::Get) + | (Self::Values(_), CacheOp::BatchGet) + | (Self::Number(_), CacheOp::Increment) + | ( + Self::Stored, + CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage + ) + ) + } +} + +/// A fallback chain's dict targets, which can carry an api key. +#[derive(Clone, PartialEq, veil::Redact)] +pub(crate) struct DictTargets(#[redact(with = "[REDACTED]")] pub(crate) Vec>); + +/// What a call is blocked on, for Python to run. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum HostCall { + Attempt { + deployment_id: String, + /// The picked deployment as configured: `model_name`, `litellm_params`, `model_info` + /// (with its `id`). The host builds the request from it; it never looks the deployment up + /// in a Python router. + deployment: Value, + /// The name the attempt resolved: the requested name or the fallback target. + model: String, + model_group: String, + /// The `order` level of a same-group order fallback. + target_order: Option, + /// Every deployment `model` resolved to for this caller, the picked one included. + candidate_ids: Vec, + retry: u32, + fallback_depth: u32, + fallback_level: u32, + timeout: Option, + /// Request params the fallback walk's dict targets merged in. + params: Option>, + /// The walk level whose request metadata the attempt carries. + scope: u32, + max_retries: u32, + model_group_size: usize, + }, + Sleep { + seconds: f64, + }, + AllowFallback { + target: String, + }, + RefuseFallbacks { + /// The walk level whose chain the targets are. + level: u32, + targets: DictTargets, + }, + Select { + stage: Stage, + model_group: String, + /// The name the request resolved from, which strategies key their state by. + name: String, + /// Whose selector a strategy reads: `default`, `group` (with `group`) or `override`. + scope: &'static str, + group: Option, + /// The deployments the name resolved to, before health filters (the tag stage reads them). + pool: Vec, + candidates: Vec, + }, + /// Pre-routing before a pick resolves `model`: answered with the model group to resolve + /// instead (a string), a custom strategy's deployment document (a mapping), or `None`. + PreRoute { + model: String, + fallback_depth: u32, + retry: u32, + }, + /// One [`RouterCache`] op against the host's cache. `key` is set for every op but + /// `batch_get`, which sets `keys`; `value` is set for `set` (the JSON to store) and + /// `increment` (the amount); `ttl` is in seconds. + Cache { + op: CacheOp, + key: Option, + keys: Option>, + value: Option, + ttl: Option, + /// The op reads or writes this worker's in-memory tier only (`local_only=True`). + local: bool, + }, + /// The deployment build's `os.environ/` references: answered with one value per ref. + Secrets { + index: usize, + refs: Vec, + }, + /// The deployment build's `validate`, `provider` and `added` steps for one deployment. + Build { + kind: StepKind, + path: BuildPath, + index: usize, + entry: Value, + declared_id: Option, + duplicate_id: bool, + /// The provider the `provider` step resolved (`added` only). + provider: Option, + }, + /// One lookup of a model-info query ([`ModelInfoLookup`], as its `lookup` field and + /// arguments): answered as the lookup documents. + ModelInfo { + lookup: Value, + }, +} + +impl HostCall { + pub(crate) fn kind(&self) -> StepKind { + match self { + Self::Attempt { .. } => StepKind::Attempt, + Self::Sleep { .. } => StepKind::Sleep, + Self::AllowFallback { .. } => StepKind::AllowFallback, + Self::RefuseFallbacks { .. } => StepKind::RefuseFallbacks, + Self::Select { .. } => StepKind::Select, + Self::PreRoute { .. } => StepKind::PreRoute, + Self::Cache { .. } => StepKind::Cache, + Self::Secrets { .. } => StepKind::Secrets, + Self::Build { kind, .. } => *kind, + Self::ModelInfo { .. } => StepKind::ModelInfo, + } + } +} + +/// Python's response to a [`HostCall`]. +#[derive(Debug)] +pub(crate) enum HostCallResponse { + /// The attempt's total tokens, or its failure. + Attempt(Result), + /// A successful attempt whose response the provider blocked, or a stream whose outcome the + /// host reports when it ends (`deferred`), with its tokens. + Served { + total_tokens: u64, + blocked: Option, + deferred: bool, + tracks_usage: Option, + }, + Slept, + AllowFallback(bool), + /// Whether the chain a `refuse_fallbacks` step named is refused. + RefuseFallbacks(bool), + /// The candidate ids that remain, pick first. + Select(Vec), + /// The model group to resolve instead, a custom strategy's deployment, or `None`. + PreRoute(Option), + /// The facts a `pre_call` select step asked for. + PreCall(PreCallFacts), + /// The candidate models a `routing_plugins` select step reported. + Models(Option>), + /// The ids a `callback_filters` select step left, or the error a filter raised. + Filtered(Result, AttemptError>), + /// What a pick's select or pre-route step, or a fallback check, raised on the host instead of + /// answering: the walk takes it as the try's failure. + Raised(AttemptError), + Cache(CacheOutcome), + /// One value per ref of a `secrets` step. + Secrets(Vec), + Validated(Built), + Provided(Provided), + Added(Built), + /// A `model_info` lookup's response, as the host spelled it. + ModelInfo(Value), +} + +impl HostCallResponse { + fn kind(&self) -> StepKind { + match self { + Self::Attempt(_) | Self::Served { .. } => StepKind::Attempt, + // Answers whichever step raised it (`check`). + Self::Raised(_) => StepKind::Select, + Self::Slept => StepKind::Sleep, + Self::AllowFallback(_) => StepKind::AllowFallback, + Self::RefuseFallbacks(_) => StepKind::RefuseFallbacks, + Self::Select(_) | Self::PreCall(_) | Self::Models(_) | Self::Filtered(_) => { + StepKind::Select + } + Self::PreRoute(_) => StepKind::PreRoute, + Self::Cache(_) => StepKind::Cache, + Self::Secrets(_) => StepKind::Secrets, + Self::Validated(_) => StepKind::Validate, + Self::Provided(_) => StepKind::Provider, + Self::Added(_) => StepKind::Added, + Self::ModelInfo(_) => StepKind::ModelInfo, + } + } + + fn check(&self, call: &HostCall) -> Result<(), DriveError> { + if matches!(self, Self::Raised(_)) + && matches!( + call.kind(), + StepKind::Select | StepKind::PreRoute | StepKind::AllowFallback + ) + { + return Ok(()); + } + if self.kind() != call.kind() { + return Err(DriveError::WrongAnswer { + expected: call.kind(), + got: self.kind(), + }); + } + if let (Self::Select(ids) | Self::Filtered(Ok(ids)), HostCall::Select { candidates, .. }) = + (self, call) + && let Some(id) = ids.iter().find(|id| !candidates.contains(id)) + { + return Err(DriveError::UnknownCandidate { id: id.clone() }); + } + if let (Self::Secrets(values), HostCall::Secrets { refs, .. }) = (self, call) + && values.len() != refs.len() + { + return Err(DriveError::SecretsLength { + expected: refs.len(), + got: values.len(), + }); + } + if let (Self::Cache(outcome), HostCall::Cache { op, keys, .. }) = (self, call) { + if !outcome.answers(*op) { + return Err(DriveError::WrongCacheOutcome { op: *op }); + } + if let (CacheOutcome::Values(values), Some(keys)) = (outcome, keys) + && values.len() != keys.len() + { + return Err(DriveError::BatchLength { + expected: keys.len(), + got: values.len(), + }); + } + } + Ok(()) + } +} + +/// How a driven future ended well. +#[derive(Debug)] +pub(crate) enum Finish { + /// `start`: the call's response. + Call(RouterResponse), + /// `select`: the deployment picked and the model group it serves. + Selected { + deployment_id: String, + model_group: String, + /// The deployment as served: a priority group's member carries its priority as `order`. + deployment: Value, + }, + /// `healthy`: the deployments a pick would choose among, as served. + Healthy { + /// A deployment named by its id, which the listing returns alone rather than as a list. + single: bool, + deployment_ids: Vec, + deployments: Vec, + }, + /// `report_failure`: the cooldown applied, if the deployment was cooled down. + Reported { cooldown: Option }, + /// `report_success` / `count_success`: the success counter's key, if one was counted. + Counted { key: Option }, + /// `build`: the deployments it installed, in order. + Built(Vec), + /// `router_model_info`: the merged model info (`None` when the lookup answered `None`). + RouterModelInfo(Option), + /// `deployment_model_info`: the merged model info, if any. + DeploymentModelInfo(Option>), + /// A model-info query that ends with one JSON answer (`model_group_info`, + /// `configured_token_limits`, `model_listing_info`). + Value(Value), +} + +/// One step of a call: what it is blocked on, or how it ended. +#[derive(Debug)] +pub(crate) enum Step { + HostCall(HostCall), + Done(Finish), + Failed(Error), +} + +/// A misuse of the step protocol by the caller. +#[derive(Debug, thiserror::Error)] +pub(crate) enum DriveError { + #[error("the router call already finished")] + Finished, + #[error("the router call has not made a host call yet, so the outcome must be None")] + UnexpectedResponse, + #[error("the router call is waiting for an outcome of its `{expected}` step")] + MissingResponse { expected: StepKind }, + #[error("the router call is waiting for an outcome of its `{expected}` step, not `{got}`")] + WrongAnswer { expected: StepKind, got: StepKind }, + #[error("deployment `{id}` was not one of the select step's candidates")] + UnknownCandidate { id: String }, + #[error("the outcome does not answer a `{op}` cache step")] + WrongCacheOutcome { op: CacheOp }, + #[error("a `batch_get` cache step for {expected} keys was answered with {got} values")] + BatchLength { expected: usize, got: usize }, + #[error("a `secrets` step for {expected} references was answered with {got} values")] + SecretsLength { expected: usize, got: usize }, + #[error("the router call is pending on something other than a step")] + Stalled, +} + +impl DriveError { + /// Whether the caller passed a bad outcome, rather than driving a call that cannot go on. + pub(crate) fn is_request(&self) -> bool { + match self { + Self::UnexpectedResponse + | Self::MissingResponse { .. } + | Self::WrongAnswer { .. } + | Self::UnknownCandidate { .. } + | Self::WrongCacheOutcome { .. } + | Self::BatchLength { .. } + | Self::SecretsLength { .. } => true, + Self::Finished | Self::Stalled => false, + } + } +} + +/// The host call a router call is blocked on and the response that releases it. +#[derive(Default)] +struct Exchange { + call: Option, + response: Option, + /// Events the call reported since the last step, handed to Python with the next one. + events: Vec, +} + +type Shared = Arc>; + +fn lock(exchange: &Mutex) -> MutexGuard<'_, Exchange> { + exchange.lock().unwrap_or_else(PoisonError::into_inner) +} + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +/// Binds an exchange to this thread for one poll and restores the previous one after. +struct Bound(Option); + +impl Bound { + fn new(exchange: &Shared) -> Self { + Self(CURRENT.replace(Some(exchange.clone()))) + } +} + +impl Drop for Bound { + fn drop(&mut self) { + CURRENT.set(self.0.take()); + } +} + +/// Park the host call `call` in the bound exchange and wait for its response. +/// `response`, or the failure a step answered with instead ([`HostCallResponse::Raised`]). +fn answered(response: HostCallResponse) -> Result { + match response { + HostCallResponse::Raised(failure) => Err(failure), + response => Ok(response), + } +} + +async fn call_host(call: HostCall) -> HostCallResponse { + let mut call = Some(call); + poll_fn(move |_| { + CURRENT.with_borrow(|current| { + let Some(exchange) = current else { + return Poll::Pending; + }; + let mut exchange = lock(exchange); + if let Some(call) = call.take() { + exchange.call = Some(call); + return Poll::Pending; + } + exchange.response.take().map_or(Poll::Pending, Poll::Ready) + }) + }) + .await +} + +const CHECKED: &str = "the driver checks each response against its host call"; + +fn ttl_seconds(ttl: Option) -> Option { + ttl.map(|ttl| ttl.as_secs_f64()) +} + +/// An increment amount as Python's counters store it: an int when it is whole. +fn amount_value(amount: f64) -> Value { + if amount.fract() == 0.0 && amount.abs() < i64::MAX as f64 { + Value::from(amount as i64) + } else { + Value::from(amount) + } +} + +/// The host's cache: every op is a `cache` step Python runs against the router's `DualCache`. +pub(crate) struct StepCache; + +async fn cache_step( + op: CacheOp, + key: Option<&str>, + keys: Option<&[String]>, + value: Option, + ttl: Option, +) -> CacheOutcome { + call_cache(op, key, keys, value, ttl, false).await +} + +async fn call_cache( + op: CacheOp, + key: Option<&str>, + keys: Option<&[String]>, + value: Option, + ttl: Option, + local: bool, +) -> CacheOutcome { + let call = HostCall::Cache { + op, + key: key.map(str::to_owned), + keys: keys.map(<[String]>::to_vec), + value, + ttl: ttl_seconds(ttl), + local, + }; + let HostCallResponse::Cache(outcome) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + outcome +} + +#[async_trait] +impl RouterCache for StepCache { + async fn async_get_cache(&self, key: &str) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = + cache_step(CacheOp::Get, Some(key), None, None, None).await + else { + unreachable!("{CHECKED}"); + }; + Ok(value) + } + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), litellm_cache::Error> { + cache_step(CacheOp::Set, Some(key), None, Some(value), ttl).await; + Ok(()) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + let value = Some(amount_value(amount)); + let CacheOutcome::Number(next) = + cache_step(CacheOp::Increment, Some(key), None, value, ttl).await + else { + unreachable!("{CHECKED}"); + }; + Ok(next) + } + + async fn async_batch_get_cache( + &self, + keys: &[String], + ) -> Result>, litellm_cache::Error> { + let CacheOutcome::Values(values) = + cache_step(CacheOp::BatchGet, None, Some(keys), None, None).await + else { + unreachable!("{CHECKED}"); + }; + Ok(values) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), litellm_cache::Error> { + cache_step(CacheOp::Delete, Some(key), None, None, None).await; + Ok(()) + } + + async fn async_increment_usage( + &self, + increments: &[(String, f64)], + ttl: Option, + ) -> Result<(), litellm_cache::Error> { + let keys: Vec = increments.iter().map(|(key, _)| key.clone()).collect(); + let amounts = increments + .iter() + .map(|(_, amount)| amount_value(*amount)) + .collect(); + let value = Some(Value::Array(amounts)); + cache_step(CacheOp::IncrementUsage, None, Some(&keys), value, ttl).await; + Ok(()) + } + + async fn async_get_local(&self, key: &str) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = + call_cache(CacheOp::Get, Some(key), None, None, None, true).await + else { + unreachable!("{CHECKED}"); + }; + Ok(value) + } + + async fn async_increment_local( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + let value = Some(amount_value(amount)); + let CacheOutcome::Number(next) = + call_cache(CacheOp::Increment, Some(key), None, value, ttl, true).await + else { + unreachable!("{CHECKED}"); + }; + Ok(next) + } +} + +/// A deployment as Python spells a `model_list` entry, with its id in `model_info`. +pub(crate) fn deployment_record(deployment: &Deployment) -> Value { + deployment.record() +} + +/// The process's `random` module: a Python router draws its shuffle and retry jitter there, so +/// a seeded process picks the same deployments on either core. The bare module functions share +/// one generator, so these calls interleave with every other draw the process makes. +pub(crate) struct PythonRandomness; + +impl Randomness for PythonRandomness { + fn next_f64(&self) -> f64 { + Python::attach(|py| -> PyResult { + py.import("random")?.call_method0("random")?.extract() + }) + .unwrap_or_else(|_| ThreadRandomness.next_f64()) + } + + fn index_below(&self, len: usize) -> usize { + // `random.choice` is `seq[self._randbelow(len(seq))]`, and so is `randrange(len)`. + Python::attach(|py| -> PyResult { + py.import("random")? + .call_method1("randrange", (len,))? + .extract() + }) + .unwrap_or_else(|_| ThreadRandomness.index_below(len)) + } +} + +/// The [`Host`] every `RouterCoreHandle` router runs on. +pub(crate) struct StepHost; + +#[async_trait] +impl Host for StepHost { + async fn invoke(&self, attempt: &Attempt) -> Result { + let call = HostCall::Attempt { + deployment_id: attempt.deployment.id.clone(), + deployment: deployment_record(&attempt.deployment), + model: attempt.name.clone(), + model_group: attempt.model_group.clone(), + target_order: attempt.target_order, + candidate_ids: attempt.candidate_ids.clone(), + retry: attempt.retry, + fallback_depth: attempt.fallback_depth, + fallback_level: attempt.fallback_level, + timeout: attempt.timeout.map(|timeout| timeout.as_secs_f64()), + params: attempt + .params + .as_ref() + .map(|params| Box::new(Value::Object(Map::clone(params)))), + scope: attempt.scope, + max_retries: attempt.max_retries, + model_group_size: attempt.model_group_size, + }; + let (result, blocked, deferred, tracks_usage) = match call_host(call).await { + HostCallResponse::Attempt(result) => (result, None, false, None), + HostCallResponse::Served { + total_tokens, + blocked, + deferred, + tracks_usage, + } => (Ok(total_tokens), blocked, deferred, tracks_usage), + _ => unreachable!("{CHECKED}"), + }; + result.map(|total_tokens| Output { + value: Payload(Arc::new(())), + total_tokens, + blocked, + deferred, + tracks_usage, + }) + } + + async fn select( + &self, + stage: &SelectStage, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: match stage { + SelectStage::Plugins => Stage::Plugins, + SelectStage::Strategy(strategy) => Stage::Strategy(strategy.to_string()), + SelectStage::Tags => Stage::Tags, + SelectStage::TextOnly => Stage::TextOnly, + SelectStage::RouteIncapable { handler } => Stage::RouteIncapable { + handler: handler.clone(), + }, + SelectStage::MediaRefusing => Stage::MediaRefusing, + }, + pool: ctx.pool.iter().map(|d| d.id.clone()).collect(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::Select(ids) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(ids + .iter() + .filter_map(|id| candidates.iter().find(|d| d.id == *id).cloned()) + .collect()) + } + + async fn pre_route(&self, pre: &PreRoute<'_>) -> Result, Error> { + let call = HostCall::PreRoute { + model: pre.name.to_owned(), + fallback_depth: pre.fallback_depth, + retry: pre.retry, + }; + let HostCallResponse::PreRoute(routed) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(routed) + } + + async fn routing_plugin_models( + &self, + ctx: &RoutingContext<'_>, + run: bool, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::RoutingPlugins { run }, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: Vec::new(), + }; + let HostCallResponse::Models(models) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(models) + } + + async fn provider_request(&self, ctx: &RoutingContext<'_>) -> Result, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::ProviderRequest, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: Vec::new(), + }; + let HostCallResponse::Models(models) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(models.into_iter().flatten().next()) + } + + async fn pre_call_facts( + &self, + ctx: &RoutingContext<'_>, + candidates: &[Arc], + ) -> Result { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::PreCall, + pool: ctx.pool.iter().map(|d| d.id.clone()).collect(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::PreCall(facts) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(facts) + } + + async fn callback_filters( + &self, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let joined = |own: &[String], walked: &[String]| [own, walked].concat(); + let call = HostCall::Select { + stage: Stage::CallbackFilters { + excluded_ids: joined(&ctx.request.excluded_ids, ctx.excluded_ids), + skipped_ids: joined(&ctx.request.retry_skipped_ids, ctx.skipped_ids), + target_order: ctx.target_order, + }, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::Filtered(result) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + let ids = result.map_err(Error::Attempt)?; + Ok(ids + .iter() + .filter_map(|id| candidates.iter().find(|d| d.id == *id).cloned()) + .collect()) + } + + async fn allow_fallback( + &self, + _request: &RouterRequest, + target: &str, + ) -> Result { + let call = HostCall::AllowFallback { + target: target.to_owned(), + }; + let HostCallResponse::AllowFallback(allowed) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(allowed) + } + + async fn refuses_fallbacks(&self, level: u32, targets: &[Map]) -> bool { + let call = HostCall::RefuseFallbacks { + level, + targets: DictTargets(targets.to_vec()), + }; + let HostCallResponse::RefuseFallbacks(refused) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + refused + } + + /// A zero sleep is asked too: Python's retry loop awaits `asyncio.sleep(0)` between sibling + /// retries, which yields to the event loop. + async fn sleep(&self, duration: Duration) { + let call = HostCall::Sleep { + seconds: duration.as_secs_f64(), + }; + let HostCallResponse::Slept = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + } + + /// Kept for Python only when it acts on it: an exhausted retry loop is stamped on the + /// error Python raises, a fallback hop tells it which target a failed walk ended on, and a + /// cooldown or a pick is reported to the host's Prometheus and service loggers (a pick also + /// joins the router cache's per-deployment count, a failed walk the router's error log). + fn observe(&self, event: &RouterEvent) { + if !matches!( + event, + RouterEvent::RetriesExhausted { .. } + | RouterEvent::Retry { .. } + | RouterEvent::Fallback { .. } + | RouterEvent::FallingBack { .. } + | RouterEvent::OrderChainFailed { .. } + | RouterEvent::FallbacksFailed { .. } + | RouterEvent::LevelFailed { .. } + | RouterEvent::PickFailed { .. } + | RouterEvent::RetriesOpened { .. } + | RouterEvent::RetryBudget { .. } + | RouterEvent::MockFailed { .. } + | RouterEvent::RetryLogged { .. } + | RouterEvent::RetryPolicyRaised { .. } + | RouterEvent::RetryCountRaised { .. } + | RouterEvent::FallbackDone { .. } + | RouterEvent::Blocked { .. } + | RouterEvent::CooldownSet { .. } + | RouterEvent::Selected { .. } + ) { + return; + } + CURRENT.with_borrow(|current| { + if let Some(exchange) = current { + lock(exchange).events.push(event.clone()); + } + }); + } + + fn cache(&self) -> Arc { + Arc::new(StepCache) + } + + async fn secrets(&self, index: usize, refs: &[SecretRef]) -> Result, Error> { + let HostCallResponse::Secrets(values) = call_host(HostCall::Secrets { + index, + refs: refs.to_vec(), + }) + .await + else { + unreachable!("{CHECKED}"); + }; + Ok(values) + } + + async fn validate(&self, build: &BuildAsk<'_>) -> Result { + let HostCallResponse::Validated(built) = + call_host(build_host_call(StepKind::Validate, build, None)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(built) + } + + async fn provider(&self, build: &BuildAsk<'_>) -> Result { + let HostCallResponse::Provided(provided) = + call_host(build_host_call(StepKind::Provider, build, None)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(provided) + } + + async fn added(&self, build: &BuildAsk<'_>, provider: Option<&str>) -> Result { + let HostCallResponse::Added(built) = + call_host(build_host_call(StepKind::Added, build, provider)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(built) + } + + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + let lookup = serde_json::to_value(lookup).unwrap_or(Value::Null); + let HostCallResponse::ModelInfo(response) = call_host(HostCall::ModelInfo { lookup }).await + else { + unreachable!("{CHECKED}"); + }; + Ok(response) + } +} + +fn build_host_call(kind: StepKind, build: &BuildAsk<'_>, provider: Option<&str>) -> HostCall { + HostCall::Build { + kind, + path: build.path, + index: build.index, + entry: Value::Object(build.entry.clone()), + declared_id: build.declared_id.map(str::to_owned), + duplicate_id: build.duplicate_id, + provider: provider.map(str::to_owned), + } +} + +type CallFuture = Pin> + Send>>; + +/// Poll `future` once with `exchange` bound. +fn poll_bound(future: Pin<&mut F>, exchange: &Shared) -> Poll { + let _bound = Bound::new(exchange); + future.poll(&mut Context::from_waker(Waker::noop())) +} + +/// Drives one router call a step at a time. +pub(crate) struct Driver { + future: Option, + exchange: Shared, + pending_call: Option, +} + +impl Driver { + pub(crate) fn new(call: impl Future> + Send + 'static) -> Self { + Self { + future: Some(Box::pin(call)), + exchange: Shared::default(), + pending_call: None, + } + } + + /// The host call the router call is blocked on, if it made one. + pub(crate) fn pending_call(&self) -> Option<&HostCall> { + self.pending_call.as_ref() + } + + /// The events reported since the last call. + pub(crate) fn take_events(&self) -> Vec { + std::mem::take(&mut lock(&self.exchange).events) + } + + /// Hand back the response to the pending host call (`None` on the first call) and run the call to + /// its next step or its end. + pub(crate) fn resume( + &mut self, + response: Option, + ) -> Result { + let Some(future) = self.future.as_mut() else { + return Err(DriveError::Finished); + }; + match (&self.pending_call, response) { + (None, None) => {} + (None, Some(_)) => return Err(DriveError::UnexpectedResponse), + (Some(call), None) => { + return Err(DriveError::MissingResponse { + expected: call.kind(), + }); + } + (Some(call), Some(response)) => { + response.check(call)?; + lock(&self.exchange).response = Some(response); + } + } + self.pending_call = None; + match poll_bound(future.as_mut(), &self.exchange) { + Poll::Ready(result) => { + self.future = None; + Ok(match result { + Ok(finish) => Step::Done(finish), + Err(error) => Step::Failed(error), + }) + } + Poll::Pending => { + let Some(call) = lock(&self.exchange).call.take() else { + self.future = None; + return Err(DriveError::Stalled); + }; + self.pending_call = Some(call.clone()); + Ok(Step::HostCall(call)) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, + }; + + use litellm_router_core::{ + FailureKind, Op, Router, RouterConfig, RouterDynamicConfig, SeededRandomness, + }; + use rstest::{fixture, rstest}; + use serde_json::{Value, json}; + + use super::*; + + fn router(config: Value, dynamic: Value) -> Router { + let config: RouterConfig = serde_json::from_value(config).unwrap(); + let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap(); + Router::builder(config, dynamic, Arc::new(StepHost)) + .randomness(Arc::new(SeededRandomness::new(7))) + .build() + .unwrap() + } + + fn deployment(group: &str, id: &str) -> Value { + json!({"model_name": group, "litellm_params": {"model": format!("openai/{id}")}, "model_info": {"id": id}}) + } + + fn start(router: &Router, model: &str) -> Driver { + let router = router.clone(); + let request = RouterRequest { + model: model.to_owned(), + ..RouterRequest::default() + }; + Driver::new(async move { + router + .call(Op::from("acompletion"), request) + .await + .map(Finish::Call) + }) + } + + fn rate_limited() -> HostCallResponse { + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::RateLimit, Some(429)))) + } + + /// A dict-backed stand-in for Python's `DualCache` that answers `cache` steps and keeps + /// every one it answered. + #[derive(Default)] + struct Store { + entries: HashMap, + calls: Vec, + } + + impl Store { + fn respond(&mut self, call: &HostCall) -> HostCallResponse { + let HostCall::Cache { + op, + key, + keys, + value, + .. + } = call + else { + panic!("not a cache step: {call:?}"); + }; + self.calls.push(call.clone()); + let key = key.clone().unwrap_or_default(); + HostCallResponse::Cache(match op { + CacheOp::Get => CacheOutcome::Value(self.entries.get(&key).cloned()), + CacheOp::Set => { + self.entries.insert(key, value.clone().unwrap()); + CacheOutcome::Stored + } + CacheOp::Increment => { + let current = self.entries.get(&key).and_then(Value::as_f64); + let next = + current.unwrap_or(0.0) + value.as_ref().and_then(Value::as_f64).unwrap(); + self.entries.insert(key, amount_value(next)); + CacheOutcome::Number(next) + } + CacheOp::IncrementUsage => { + let amounts = value.as_ref().and_then(Value::as_array).unwrap(); + for (key, amount) in keys.iter().flatten().zip(amounts) { + let current = self.entries.get(key).and_then(Value::as_f64); + let next = current.unwrap_or(0.0) + amount.as_f64().unwrap(); + self.entries.insert(key.clone(), amount_value(next)); + } + CacheOutcome::Stored + } + CacheOp::BatchGet => CacheOutcome::Values( + keys.iter() + .flatten() + .map(|key| self.entries.get(key).cloned()) + .collect(), + ), + CacheOp::Delete => { + self.entries.remove(&key); + CacheOutcome::Stored + } + }) + } + + /// Resume with `response`, answering every cache step on the way, up to the next other + /// step. + fn resume(&mut self, driver: &mut Driver, response: Option) -> Step { + let mut step = driver.resume(response).unwrap(); + while let Step::HostCall(call @ HostCall::Cache { .. }) = &step { + let response = self.respond(call); + step = driver.resume(Some(response)).unwrap(); + } + step + } + } + + #[fixture] + fn pair() -> Router { + router( + json!({"model_list": [deployment("primary", "a1"), deployment("primary", "a2")]}), + json!({}), + ) + } + + #[rstest] + fn a_first_attempt_success_is_done(pair: Router) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + let Step::HostCall(HostCall::Attempt { + deployment_id, + model_group, + retry: 0, + fallback_depth: 0, + .. + }) = store.resume(&mut driver, None) + else { + panic!("the call should start with an attempt"); + }; + assert_eq!(model_group, "primary"); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(12)))) + else { + panic!("a successful attempt should finish the call"); + }; + assert_eq!(response.deployment_id, deployment_id); + assert_eq!(response.output.total_tokens, 12); + assert_eq!( + (response.attempted_retries, response.attempted_fallbacks), + (0, 0) + ); + assert!(matches!(driver.resume(None), Err(DriveError::Finished))); + } + + #[rstest] + fn cooldown_reads_are_cache_steps(pair: Router) { + let mut driver = start(&pair, "primary"); + let step = driver.resume(None).unwrap(); + let Step::HostCall(HostCall::Cache { + op: CacheOp::BatchGet, + key: None, + keys: Some(keys), + value: None, + ttl: None, + local: false, + }) = step + else { + panic!("the call should read cooldowns first, got {step:?}"); + }; + assert_eq!(keys, ["deployment:a1:cooldown", "deployment:a2:cooldown"]); + } + + #[rstest] + #[case::lone_deployment_backs_off(vec![deployment("primary", "a1")], true)] + #[case::sibling_retries_at_once( + vec![deployment("primary", "a1"), deployment("primary", "a2")], + false + )] + fn a_rate_limit_retries_after_the_backoff( + #[case] model_list: Vec, + #[case] backs_off: bool, + ) { + let router = router(json!({"model_list": model_list}), json!({"num_retries": 1})); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + assert_eq!( + store.resume(&mut driver, None).unwrap_kind(), + StepKind::Attempt + ); + let step = store.resume(&mut driver, Some(rate_limited())); + let Step::HostCall(HostCall::Sleep { seconds }) = step else { + panic!("a retry should sleep first, got {step:?}"); + }; + assert_eq!(seconds > 0.0, backs_off, "slept {seconds}s"); + let step = store.resume(&mut driver, Some(HostCallResponse::Slept)); + assert!( + matches!(step, Step::HostCall(HostCall::Attempt { retry: 1, .. })), + "expected the retry, got {step:?}" + ); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(1)))) + else { + panic!("the retry should finish the call"); + }; + assert_eq!(response.attempted_retries, 1); + } + + #[rstest] + fn a_cooldown_python_wrote_is_read_back_by_the_next_call() { + let pair_without_retries = router( + json!({"model_list": [deployment("primary", "a1"), deployment("primary", "a2")]}), + json!({"num_retries": 0}), + ); + let mut store = Store::default(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + store.entries.insert( + "deployment:a1:cooldown".into(), + json!({"exception_received": "slow down", "status_code": "429", "timestamp": now, "cooldown_time": 60}), + ); + for _ in 0..8 { + let mut call = start(&pair_without_retries, "primary"); + let Step::HostCall(HostCall::Attempt { deployment_id, .. }) = + store.resume(&mut call, None) + else { + panic!("the call should attempt the healthy deployment"); + }; + assert_eq!(deployment_id, "a2", "the cooled-down deployment was picked"); + } + } + + /// The host only runs the attempt; the core records its outcome (cooldowns, fail counters, + /// usage) through the host's cache. + #[rstest] + #[case::failure(rate_limited())] + #[case::success(HostCallResponse::Attempt(Ok(7)))] + fn attempt_outcomes_are_recorded_by_the_core(#[case] outcome: HostCallResponse) { + let tracked = |id: &str| { + json!({ + "model_name": "primary", + "litellm_params": {"model": format!("openai/{id}"), "tpm": 100, "cooldown_time": 7}, + "model_info": {"id": id, "allowed_fails": 3}, + }) + }; + let router = router( + json!({"model_list": [tracked("a1"), tracked("a2")]}), + json!({"num_retries": 0}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let step = store.resume(&mut driver, Some(outcome)); + assert!( + matches!(step, Step::Done(_) | Step::Failed(_)), + "got {step:?}" + ); + let writes: Vec<&HostCall> = store + .calls + .iter() + .filter(|call| { + matches!(call, HostCall::Cache { op, .. } if !matches!(op, CacheOp::Get | CacheOp::BatchGet)) + }) + .collect(); + assert!(!writes.is_empty(), "the core wrote no outcome"); + } + + #[rstest] + fn a_failed_group_falls_back_once_python_allows_it() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": ["backup"]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + assert_eq!( + store.resume(&mut driver, None).unwrap_kind(), + StepKind::Attempt + ); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + let step = store.resume(&mut driver, Some(bad_request)); + assert!( + matches!(&step, Step::HostCall(HostCall::AllowFallback { target }) if target == "backup"), + "expected the fallback check, got {step:?}" + ); + let step = store.resume(&mut driver, Some(HostCallResponse::AllowFallback(true))); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment_id, fallback_depth: 1, .. }) if deployment_id == "b1"), + "expected the fallback attempt, got {step:?}" + ); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(3)))) + else { + panic!("the fallback should finish the call"); + }; + assert_eq!(response.model_group, "backup"); + assert_eq!(response.attempted_fallbacks, 1); + } + + #[rstest] + fn a_refused_fallback_fails_with_the_attempt_error() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": ["backup"]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + store.resume(&mut driver, Some(bad_request)); + let Step::Failed(error) = + store.resume(&mut driver, Some(HostCallResponse::AllowFallback(false))) + else { + panic!("a refused fallback should fail the call"); + }; + assert_eq!( + error.attempt().map(|a| a.kind), + Some(FailureKind::BadRequest) + ); + } + + #[rstest] + #[case::none_while_waiting(None, StepKind::Attempt)] + #[case::wrong_kind(Some(HostCallResponse::Slept), StepKind::Attempt)] + #[case::cache_outcome(Some(HostCallResponse::Cache(CacheOutcome::Stored)), StepKind::Attempt)] + fn a_mismatched_outcome_is_rejected_and_the_step_kept( + pair: Router, + #[case] response: Option, + #[case] expected: StepKind, + ) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + store.resume(&mut driver, None); + let error = driver.resume(response).unwrap_err(); + assert!(error.is_request(), "{error}"); + assert_eq!(driver.pending_call().map(HostCall::kind), Some(expected)); + assert!(matches!( + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(1)))), + Step::Done(_) + )); + } + + #[rstest] + #[case::wrong_op(HostCallResponse::Cache(CacheOutcome::Stored))] + #[case::number_for_batch(HostCallResponse::Cache(CacheOutcome::Number(1.0)))] + #[case::short_batch(HostCallResponse::Cache(CacheOutcome::Values(vec![None])))] + #[case::not_a_cache_outcome(HostCallResponse::Slept)] + fn a_mismatched_cache_outcome_is_rejected_and_the_step_kept( + pair: Router, + #[case] response: HostCallResponse, + ) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + let step = driver.resume(None).unwrap(); + let Step::HostCall( + call @ HostCall::Cache { + op: CacheOp::BatchGet, + .. + }, + ) = &step + else { + panic!("expected the cooldown read, got {step:?}"); + }; + let error = driver.resume(Some(response)).unwrap_err(); + assert!(error.is_request(), "{error}"); + assert_eq!(driver.pending_call(), Some(call)); + let response = store.respond(call); + assert_eq!( + store.resume(&mut driver, Some(response)).unwrap_kind(), + StepKind::Attempt + ); + } + + #[rstest] + fn a_strategy_python_runs_asks_for_a_selection_and_keeps_the_pick() { + let router = router( + json!({ + "model_list": [deployment("primary", "a1"), deployment("primary", "a2")], + "routing_strategy": "least-busy", + }), + json!({}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + let step = store.resume(&mut driver, None); + let Step::HostCall(HostCall::Select { + stage: Stage::Strategy(strategy), + candidates, + .. + }) = step + else { + panic!("a host strategy should ask for a selection, got {step:?}"); + }; + assert_eq!(strategy, "least-busy"); + assert_eq!(candidates, ["a1", "a2"]); + let step = store.resume( + &mut driver, + Some(HostCallResponse::Select(vec!["a2".to_owned()])), + ); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment_id, .. }) if deployment_id == "a2"), + "expected the picked deployment, got {step:?}" + ); + } + + #[rstest] + fn a_selection_is_driven_through_cache_steps(pair: Router) { + let request = RouterRequest { + model: "primary".to_owned(), + ..RouterRequest::default() + }; + let mut store = Store::default(); + let mut driver = Driver::new(async move { + pair.select(&Op::from("acompletion"), &request) + .await + .map(|deployment| Finish::Selected { + deployment_id: deployment.id.clone(), + model_group: deployment.model_name.clone(), + deployment: deployment_record(&deployment), + }) + }); + let Step::Done(Finish::Selected { + deployment_id, + model_group, + .. + }) = store.resume(&mut driver, None) + else { + panic!("a selection should finish with the pick"); + }; + assert!(["a1", "a2"].contains(&deployment_id.as_str())); + assert_eq!(model_group, "primary"); + assert!(matches!( + store.calls.as_slice(), + [HostCall::Cache { + op: CacheOp::BatchGet, + .. + }] + )); + } + + #[rstest] + fn dict_targets_keep_their_api_key_out_of_debug() { + let target = json!({"model": "b", "api_key": "credential-value"}); + let Value::Object(target) = target else { + unreachable!("a JSON object"); + }; + let shown = format!("{:?}", DictTargets(vec![target])); + assert!(!shown.contains("credential-value")); + assert!(shown.contains("[REDACTED]")); + } + + #[rstest] + fn a_chain_with_a_dict_target_asks_python_before_any_hop() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": [{"model": "backup", "api_key": "k"}]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + let step = store.resume(&mut driver, Some(bad_request)); + assert!( + matches!(&step, Step::HostCall(HostCall::RefuseFallbacks { level: 0, targets }) if targets.0.len() == 1), + "expected the chain's check, got {step:?}" + ); + let Step::Failed(error) = + store.resume(&mut driver, Some(HostCallResponse::RefuseFallbacks(true))) + else { + panic!("a refused chain should fail the call"); + }; + assert_eq!( + error.attempt().map(|a| a.kind), + Some(FailureKind::BadRequest) + ); + } + + trait StepKindOf { + fn unwrap_kind(self) -> StepKind; + } + + impl StepKindOf for Step { + fn unwrap_kind(self) -> StepKind { + match self { + Step::HostCall(call) => call.kind(), + Step::Done(_) => StepKind::Done, + Step::Failed(_) => StepKind::Failed, + } + } + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/mod.rs b/litellm-rust/crates/python-bridge/src/router/mod.rs new file mode 100644 index 00000000000..c69259fcf1c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/mod.rs @@ -0,0 +1,2051 @@ +//! `RouterCoreHandle`: router-core behind a step-driven handle. Python drives each call through +//! [`RouterCall::next`], running every attempt, sleep, fallback check, selection and cache op the +//! call loop asks for and handing the outcome back; Rust never awaits Python. `select` is driven +//! the same way, since reading cooldowns is a cache step. + +mod config; +mod host; +mod retries; +mod stream; + +use std::{ + sync::{Arc, Mutex, PoisonError}, + time::{Duration, SystemTime}, +}; + +use litellm_error::Report; +use litellm_host_python::{from_py_argument, json_loads, to_py}; +use litellm_router_core::lookup::{ListScope, Lookup, Row}; +use litellm_router_core::{ + AttemptError, Blocked, BuildPath, Built, Deployment, DeploymentSpec, Error, FailureKind, + FailureReport, Op, OutcomeCallbacks, PreRouted, Provided, RetryCause, Router, RouterEvent, + SecretAt, SuccessReport, + build::Install, + health::{UsageMetric, current_minute, usage_key}, + registry::{DeploymentFilter, ResolveKind, ResolveScope}, + strategy::routing_context, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::{PyBool, PyDict, PyList, PyString}, +}; +use serde_json::Value; + +use self::host::{ + CacheOp, CacheOutcome, DriveError, Driver, Finish, HostCall, HostCallResponse, + PythonRandomness, Stage, Step, StepHost, StepKind, +}; +use crate::errors::RustBridgeDeclined; + +/// The `error_kind` of a `failed` step. +#[derive(Clone, Copy, Debug, strum::IntoStaticStr)] +enum ErrorKind { + #[strum(serialize = "config")] + Config, + #[strum(serialize = "unknown_model")] + UnknownModel, + #[strum(serialize = "no_healthy_deployments")] + NoHealthyDeployments, + #[strum(serialize = "attempt")] + Attempt, + #[strum(serialize = "cache")] + Cache, + #[strum(serialize = "retired")] + Retired, + #[strum(serialize = "paused")] + Paused, + #[strum(serialize = "not_pass_through")] + NotPassThrough, + #[strum(serialize = "no_pass_through_deployments")] + NoPassThroughDeployments, + #[strum(serialize = "only_markers")] + OnlyMarkers, + #[strum(serialize = "endpoint_not_supported")] + EndpointNotSupported, + #[strum(serialize = "ptu_reserved")] + PtuReserved, + #[strum(serialize = "context_window_exceeded")] + ContextWindowExceeded, + #[strum(serialize = "unsupported_params")] + UnsupportedParams, + #[strum(serialize = "routing_plugins_left_nothing")] + RoutingPluginsLeftNothing, + #[strum(serialize = "routing_plugins_need_async")] + RoutingPluginsNeedAsync, + #[strum(serialize = "multiple_teams")] + MultipleTeams, + #[strum(serialize = "reserved")] + Reserved, + #[strum(serialize = "invalid_router_weights")] + InvalidRouterWeights, + #[strum(serialize = "non_numeric_weight")] + NonNumericWeight, + #[strum(serialize = "missing_routing_group")] + MissingRoutingGroup, + #[strum(serialize = "default_deployment_raises")] + DefaultDeploymentRaises, + #[strum(serialize = "index_out_of_range")] + IndexOutOfRange, + #[strum(serialize = "model_info")] + ModelInfo, +} + +impl From<&Error> for ErrorKind { + fn from(error: &Error) -> Self { + match error { + Error::Config(_) => Self::Config, + Error::UnknownModel { .. } => Self::UnknownModel, + Error::NoHealthyDeployments { .. } => Self::NoHealthyDeployments, + Error::Attempt(_) => Self::Attempt, + Error::Cache(_) => Self::Cache, + Error::Retired => Self::Retired, + Error::Paused { .. } => Self::Paused, + Error::NotPassThrough { .. } => Self::NotPassThrough, + Error::NoPassThroughDeployments { .. } => Self::NoPassThroughDeployments, + Error::OnlyMarkers { .. } => Self::OnlyMarkers, + Error::EndpointNotSupported { .. } => Self::EndpointNotSupported, + Error::PtuReserved { .. } => Self::PtuReserved, + Error::ContextWindowExceeded { .. } => Self::ContextWindowExceeded, + Error::UnsupportedParams { .. } => Self::UnsupportedParams, + Error::RoutingPluginsLeftNothing { .. } => Self::RoutingPluginsLeftNothing, + Error::RoutingPluginsNeedAsync => Self::RoutingPluginsNeedAsync, + Error::MultipleTeams { .. } => Self::MultipleTeams, + Error::Reserved { .. } => Self::Reserved, + Error::InvalidRouterWeights => Self::InvalidRouterWeights, + Error::NonNumericWeight { .. } => Self::NonNumericWeight, + Error::MissingRoutingGroup { .. } => Self::MissingRoutingGroup, + Error::DefaultDeploymentRaises { .. } => Self::DefaultDeploymentRaises, + Error::IndexOutOfRange { .. } => Self::IndexOutOfRange, + Error::ModelInfo { .. } => Self::ModelInfo, + } + } +} + +fn config_error_to_pyerr(error: config::Error) -> PyErr { + let message = Report::new(&error).to_string(); + if error.declined() { + RustBridgeDeclined::new_err(message) + } else { + PyValueError::new_err(message) + } +} + +fn drive_error_to_pyerr(error: DriveError) -> PyErr { + let message = Report::new(&error).to_string(); + if error.is_request() { + PyValueError::new_err(message) + } else { + PyRuntimeError::new_err(message) + } +} + +/// A document Python passes as a mapping or as its JSON text. +fn document(value: &Bound<'_, PyAny>) -> PyResult { + match value.cast::() { + Ok(text) => { + let parsed = json_loads(value.py(), text.to_str()?.as_bytes())?; + from_py_argument(parsed.bind(value.py())) + } + Err(_) => from_py_argument(value), + } +} + +/// The `OutcomeCallbacks` a reported outcome came through; none is a report no callback makes. +fn outcome_callbacks(value: Option<&Bound<'_, PyAny>>) -> PyResult { + let Some(value) = value else { + return Ok(OutcomeCallbacks::default()); + }; + serde_json::from_value(document(value)?) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// A router-core router whose host operations Python runs. +#[pyclass(frozen)] +pub(crate) struct RouterCoreHandle { + router: Router, +} + +#[pymethods] +impl RouterCoreHandle { + #[new] + fn new(config: &Bound<'_, PyAny>, dynamic: &Bound<'_, PyAny>) -> PyResult { + let (config, dynamic) = (document(config)?, document(dynamic)?); + let build = || { + let config = config::parse_router_config(config)?; + let dynamic = config::parse_dynamic_config(dynamic)?; + Ok(Router::builder(config, dynamic, Arc::new(StepHost)) + .randomness(Arc::new(PythonRandomness)) + .build()?) + }; + build() + .map(|router| Self { router }) + .map_err(config_error_to_pyerr) + } + + /// Start a call; nothing runs until the first `next(None)`. + fn start(&self, op: &str, request: &Bound<'_, PyAny>) -> PyResult { + let op = Op::new(op); + let request = + config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router.call(op, request).await.map(Finish::Call) + })) + } + + /// Pick the deployment `model` routes to now, without calling it. Driven like `start`: its + /// steps are the cache reads and selections the pick needs, and it ends with `done` + /// (`deployment_id`, `model_group`) or `failed`. + fn select(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let deployment = router.select(&Op::from("acompletion"), &request).await?; + Ok(Finish::Selected { + deployment_id: deployment.id.clone(), + model_group: deployment.model_name.clone(), + deployment: host::deployment_record(&deployment), + }) + })) + } + + /// List the deployments a pick for `model` would choose among, driven like `select`. + fn healthy(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let candidates = router.healthy(&Op::from("acompletion"), &request).await?; + // A deployment named by its id is listed alone, as a dict (`_core_async_get_healthy_deployments`). + let single = candidates.kind.is_single() && candidates.healthy.len() == 1; + let deployments = candidates.healthy; + Ok(Finish::Healthy { + single, + deployment_ids: deployments.iter().map(|d| d.id.clone()).collect(), + deployments: deployments + .iter() + .map(|d| host::deployment_record(d)) + .collect(), + }) + })) + } + + /// Apply `{"upsert": [deployments], "delete": [ids], "settings": {dynamic document}}` and + /// return the new registry version. + fn apply(&self, changes: &Bound<'_, PyAny>) -> PyResult { + let changes = config::parse_changes(document(changes)?).map_err(config_error_to_pyerr)?; + self.router + .apply(changes) + .map_err(|error| config_error_to_pyerr(error.into())) + } + + /// Build `entries` (model_list entries) through the host, driven like `start`: its steps + /// are `secrets`, `validate`, `provider` and `added`, and it ends with `done` (`built`: one + /// `{"index", "id", "provider"}` per deployment installed). `path` is `create` + /// (`set_model_list`) or `add` (`add_deployment`); `install` is `replace`, `append` or + /// `upsert`. + #[pyo3(signature = (entries, path, resolve_secrets, install))] + fn build( + &self, + entries: &Bound<'_, PyAny>, + path: &str, + resolve_secrets: bool, + install: &str, + ) -> PyResult { + let entries: Vec> = + serde_json::from_value(document(entries)?) + .map_err(|error| PyValueError::new_err(format!("bad model_list: {error}")))?; + let path: BuildPath = path + .parse() + .map_err(|_| PyValueError::new_err(format!("`{path}` is not a build path")))?; + let install: Install = install + .parse() + .map_err(|_| PyValueError::new_err(format!("`{install}` is not a build install")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .build(path, entries, resolve_secrets, install) + .await + .map(Finish::Built) + })) + } + + /// `get_router_model_info` for the deployment `query` describes (a + /// `RouterModelInfoQuery` document), driven like `start`: its steps are `model_info` + /// lookups, and it ends with `done` (`info`: the merged map or None, `user_keys`: the keys + /// whose values are the deployment's own). + fn router_model_info(&self, query: &Bound<'_, PyAny>) -> PyResult { + let query: litellm_router_core::model_info::RouterModelInfoQuery = + serde_json::from_value(document(query)?) + .map_err(|error| PyValueError::new_err(format!("bad model-info query: {error}")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .router_model_info(&query) + .await + .map(Finish::RouterModelInfo) + })) + } + + /// `get_deployment_model_info` for deployment `model_id` serving `model_name`, driven like + /// `router_model_info`; it ends with `done` (`info`: the merged map or None). + fn deployment_model_info( + &self, + model_id: &str, + model_name: &Bound<'_, PyAny>, + ) -> PyResult { + let model_id = model_id.to_owned(); + let model_name = from_py_argument::(model_name)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .deployment_model_info(&model_id, &model_name) + .await + .map(Finish::DeploymentModelInfo) + })) + } + + /// `_set_model_group_info` for the deployments `query` names (a `GroupInfoQuery` document), + /// driven like `router_model_info`; it ends with `done` (`value`: None when no deployment + /// matched, else `{"assignments": {field: value}}`, what the merge set on the group the host + /// built from its first deployment). + fn model_group_info(&self, query: &Bound<'_, PyAny>) -> PyResult { + let query: litellm_router_core::group_info::GroupInfoQuery = + serde_json::from_value(document(query)?) + .map_err(|error| PyValueError::new_err(format!("bad group-info query: {error}")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let info = router.model_group_info(&query).await?; + Ok(Finish::Value(info.map_or( + Value::Null, + |info| serde_json::json!({"assignments": info.assignments}), + ))) + })) + } + + /// `get_configured_token_limits` for the deployment with `id` and its `configured` model_info, + /// driven like `router_model_info`; it ends with `done` (`value`: `[max_input, max_output]`). + fn configured_token_limits( + &self, + id: &Bound<'_, PyAny>, + configured: &Bound<'_, PyAny>, + ) -> PyResult { + let id = from_py_argument::(id)?; + let Value::Object(configured) = document(configured)? else { + return Err(PyValueError::new_err( + "the configured model_info must be a dict", + )); + }; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let (input, output) = router.configured_token_limits(id, &configured).await?; + Ok(Finish::Value(serde_json::json!([input, output]))) + })) + } + + /// `get_model_listing_info` over the `deployments` a listed name names, driven like + /// `router_model_info`; it ends with `done` (`value`: `{"cost_map_keys", "max_input_tokens", + /// "max_output_tokens"}`). + fn model_listing_info(&self, deployments: &Bound<'_, PyAny>) -> PyResult { + let deployments: Vec = from_py_argument(deployments)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let listing = router.model_listing_info(&deployments).await?; + Ok(Finish::Value(serde_json::json!({ + "cost_map_keys": listing.cost_map_keys, + "max_input_tokens": listing.max_input_tokens, + "max_output_tokens": listing.max_output_tokens, + }))) + })) + } + + /// `get_configured_mode` / `get_configured_display_name`: `value` when it is a string that is + /// not blank. + #[staticmethod] + fn configured_text(value: &Bound<'_, PyAny>) -> PyResult> { + let value = from_py_argument::(value).ok(); + Ok(litellm_router_core::group_info::configured_text( + value.as_ref(), + )) + } + + /// `get_configured_service_tiers`: each of `model_infos`' `service_tiers`, unvalidated. + #[staticmethod] + fn configured_service_tiers( + py: Python<'_>, + model_infos: &Bound<'_, PyAny>, + ) -> PyResult>> { + let model_infos: Vec> = from_py_argument(model_infos)?; + litellm_router_core::group_info::configured_service_tiers(&model_infos) + .iter() + .map(|tiers| to_py(py, tiers)) + .collect() + } + + /// The routing-cache key deployment `id` serving litellm `model` counts `metric` (`tpm`, + /// `rpm`, `itpm` or `otpm`) under in `minute` (`HH-MM`). + fn usage_key(&self, id: &str, model: &str, metric: &str, minute: &str) -> PyResult { + let metric: UsageMetric = metric + .parse() + .map_err(|_| PyValueError::new_err(format!("`{metric}` is not a usage metric")))?; + Ok(usage_key(id, model, metric, minute)) + } + + /// Keep the providers `other`'s build resolved for the deployments this router serves too. + fn adopt_providers(&self, other: &Self) { + self.router.adopt_providers(&other.router); + } + + fn registry_version(&self) -> u64 { + self.router.registry().version() + } + + /// The version the deployments before the newest appends last changed at: a list read at + /// or after it is current once it reads `deployments_from(len(list))`. + fn registry_rewritten(&self) -> u64 { + self.router.registry().rewritten() + } + + /// Record a failure the router did not see itself (a health check, an interceptor, a + /// logging callback), driven like `start`: its steps are cache ops, and it ends with `done` + /// (`cooled`, `cooldown_time`). `counts_request` also counts the failed request against the deployment's + /// per-minute requests, as an async call's failure callback does. `callbacks` is the logged + /// request's `OutcomeCallbacks`. + #[pyo3(signature = (id, failure, deployment_cooldown=false, count_only=false, requested_group=None, cooldown_time=None, uncounted=false, counts_request=false, usage_model=None, callbacks=None))] + #[allow(clippy::too_many_arguments)] + fn report_failure( + &self, + id: &str, + failure: &Bound<'_, PyDict>, + deployment_cooldown: bool, + count_only: bool, + requested_group: Option, + cooldown_time: Option, + uncounted: bool, + counts_request: bool, + usage_model: Option, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let failure = match attempt_outcome(failure)? { + Err(failure) => failure, + Ok(_) => { + return Err(PyValueError::new_err( + "a reported failure needs `ok: False`", + )); + } + }; + let router = self.router.clone(); + let id = id.to_owned(); + let report = FailureReport { + deployment_cooldown, + cooldown_time, + count_only, + requested_group, + uncounted, + counts_request, + usage_model, + callbacks: outcome_callbacks(callbacks)?, + }; + Ok(RouterCall::new(async move { + router + .report_failure(&id, &failure, report) + .await + .map(|cooldown| Finish::Reported { cooldown }) + })) + } + + /// Record a success a logging callback saw, driven like `report_failure`; it ends with + /// `done` (`key`: the success counter's key, or None for an unknown deployment). + /// `usage_model` is the model its usage is keyed by, and `tracks_usage` the host's answer to + /// whether it counts any (see `Output::tracks_usage`). `callbacks`: see `report_failure`. + #[pyo3(signature = (id, total_tokens, requests, usage_model=None, tracks_usage=None, callbacks=None))] + fn report_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + usage_model: Option, + tracks_usage: Option, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let router = self.router.clone(); + let id = id.to_owned(); + let report = SuccessReport { + tracks_usage, + usage_model, + callbacks: outcome_callbacks(callbacks)?, + }; + Ok(RouterCall::new(async move { + router + .report_success(&id, total_tokens, requests, report) + .await + .map(|key| Finish::Counted { key }) + })) + } + + /// Count a success against `id`, configured or not; ends with `done` (`key`). `callbacks`: + /// see `report_failure`. + #[pyo3(signature = (id, callbacks=None))] + fn count_success( + &self, + id: &str, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let router = self.router.clone(); + let id = id.to_owned(); + let callbacks = outcome_callbacks(callbacks)?; + Ok(RouterCall::new(async move { + router + .count_success(&id, callbacks) + .await + .map(|key| Finish::Counted { key }) + })) + } + + /// The usage keys of this minute for every deployment of `model_group`: + /// `[(id, tpm_key, rpm_key)]`, or None when the group has no deployment. + fn usage_keys(&self, model_group: &str) -> Option> { + let minute = current_minute(SystemTime::now()); + let group = self.router.registry().group(model_group)?; + Some( + group + .iter() + .map(|deployment| { + let model = &deployment.params.model; + ( + deployment.id.clone(), + usage_key(&deployment.id, model, UsageMetric::Tpm, &minute), + usage_key(&deployment.id, model, UsageMetric::Rpm, &minute), + ) + }) + .collect(), + ) + } + + /// Every configured deployment, as `model_list` entries with their ids, in config order. + fn deployments<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + let records = registry + .deployments(&DeploymentFilter::default()) + .iter() + .map(|deployment| to_py(py, &host::deployment_record(deployment))) + .collect::>>()?; + PyList::new(py, records) + } + + /// The deployments from position `start` on, as `deployments()` lists them. + fn deployments_from<'py>(&self, py: Python<'py>, start: usize) -> PyResult> { + let registry = self.router.registry(); + let records = registry + .deployments_from(start) + .iter() + .map(|deployment| to_py(py, &host::deployment_record(deployment))) + .collect::>>()?; + PyList::new(py, records) + } + + /// The deployment with this id, or None. + fn deployment(&self, py: Python<'_>, id: &str) -> PyResult>> { + self.router + .registry() + .deployment(id) + .map(|deployment| to_py(py, &host::deployment_record(&deployment))) + .transpose() + } + + // -- registry lookups (router-core `lookup.rs`): rows the host reads its stored entries at -- + + /// `_core_get_all_deployments`, as rows. + #[pyo3(signature = (model_name, team_id=None, rename=None))] + fn lookup_all<'py>( + &self, + py: Python<'py>, + model_name: &str, + team_id: Option<&str>, + rename: Option<&str>, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).all(model_name, team_id, rename)) + } + + /// `_team_deployments_across_teams`: the rows, and how many teams own them. + fn lookup_across_teams<'py>( + &self, + py: Python<'py>, + model: &str, + ) -> PyResult<(Bound<'py, PyAny>, usize)> { + let registry = self.router.registry(); + let (rows, teams) = Lookup::new(®istry).across_teams(model); + Ok((rows_to_py(py, &rows)?, teams)) + } + + /// `_core_config_deployments`, as rows. + fn lookup_config<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).config()) + } + + /// `_core_get_model_ids`: the ids, or the row the read raises at. + #[pyo3(signature = (model_name=None, exclude_team_models=false))] + fn lookup_model_ids<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + exclude_team_models: bool, + ) -> PyResult<(Vec, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).model_ids(model_name, exclude_team_models) { + Ok(ids) => Ok((ids, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// `_core_get_model_list`, as rows. + #[pyo3(signature = (model_name, team_id, scope))] + fn lookup_model_list<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + rows_to_py( + py, + &Lookup::new(®istry).model_list(model_name, team_id, &scope), + ) + } + + /// `_core_get_model_names`, or the row the read raises at. + fn lookup_model_names<'py>( + &self, + py: Python<'py>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(Vec, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).model_names(team_id, &scope) { + Ok(names) => Ok((names, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// `_core_get_model_access_groups`: `(group, names)` pairs in the order the groups appear, + /// or the row the read raises at. + fn lookup_access_groups<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + access_group: Option<&str>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(AccessGroups, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).access_groups(model_name, access_group, team_id, &scope) { + Ok(groups) => Ok((groups, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// Whether the whole list lists any row, and the name of its first row whose litellm model + /// is `model_id` (or ends in `/model_id` or `:model_id`); or the row the read raises at. + fn lookup_name_serving<'py>( + &self, + py: Python<'py>, + model_id: &str, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(bool, Option, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).name_serving(model_id, &scope) { + Ok((listed, name)) => Ok((listed, name, None)), + Err(raw) => Ok((false, None, Some(row_to_py(py, &raw)?))), + } + } + + /// Whether `model_names` (the group names) holds `name`. + fn has_model_name(&self, name: &str) -> bool { + self.router + .registry() + .host_indexes() + .model_names + .contains(&Value::String(name.to_owned())) + } + + /// `_core_team_model_has_alternatives`, or the row the read raises at. + fn lookup_team_model_has_alternatives<'py>( + &self, + py: Python<'py>, + id: &str, + ) -> PyResult<(bool, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).team_model_has_alternatives(id) { + Ok(found) => Ok((found, None)), + Err(raw) => Ok((false, Some(row_to_py(py, &raw)?))), + } + } + + /// The position of the group's first deployment the team may use, or the row the read + /// raises at. + fn lookup_group_usable_by_team<'py>( + &self, + py: Python<'py>, + model_group: &str, + team_id: Option<&str>, + ) -> PyResult<(Option, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).group_deployment_usable_by_team(model_group, team_id) { + Ok(position) => Ok((position, None)), + Err(raw) => Ok((None, Some(row_to_py(py, &raw)?))), + } + } + + /// The position of the team public name's first deployment, or the row the read raises at. + fn lookup_team_public_name<'py>( + &self, + py: Python<'py>, + model_id: &str, + team_id: &str, + ) -> PyResult<(Option, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).team_public_name_deployment(model_id, team_id) { + Ok(position) => Ok((position, None)), + Err(raw) => Ok((None, Some(row_to_py(py, &raw)?))), + } + } + + /// `_routable_deployments`, as rows. + fn lookup_routable<'py>( + &self, + py: Python<'py>, + model_group: &str, + team_id: Option<&str>, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).routable(model_group, team_id)) + } + + /// `_core_wildcard_route_deployments`: the team router's routes, else the global router's. + fn lookup_wildcard_route<'py>( + &self, + py: Python<'py>, + model: &str, + team_id: Option<&str>, + requests: Vec, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py( + py, + &Lookup::new(®istry).wildcard_route(model, team_id, &requests), + ) + } + + /// `_core_get_candidate_model_ids_for_route` for the resolved name: the members' ids when it + /// names a callable routing group (`members`), else for a name that is no model group the + /// team's or the wildcard route's, else the group's; or the row the read raises at. + fn lookup_candidate_ids<'py>( + &self, + py: Python<'py>, + resolved: &str, + team_id: Option<&str>, + members: Option>, + requests: Vec, + ) -> PyResult<(Vec, Option>)> { + let registry = self.router.registry(); + let lookup = Lookup::new(®istry); + match lookup.candidate_ids(resolved, team_id, members.as_deref(), &requests) { + Ok(ids) => Ok((ids, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// Whether the id map holds `id` (`_has_model_id`). + fn has_model_id(&self, id: &str) -> bool { + self.router.registry().host_indexes().has_id(id) + } + + /// What the id map holds for `id`, as written (a position, or what a caller wrote), or None. + fn id_position<'py>(&self, py: Python<'py>, id: &str) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .id_entry(id) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// The id map's entries, in order: `(id, what it holds)`. + fn id_positions<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + let pairs: Vec = registry + .host_indexes() + .id_map + .iter() + .map(|(id, held)| Value::Array(vec![Value::String(id.clone()), held.clone()])) + .collect(); + to_py(py, &Value::Array(pairs)).map(|value| value.into_bound(py)) + } + + /// What the name index holds for `name`, as written, or None when it does not index it. + fn named_positions<'py>( + &self, + py: Python<'py>, + name: &str, + ) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .named(name) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// What the team index holds for `(team, public)`, as written, or None. + fn team_positions<'py>( + &self, + py: Python<'py>, + team: &str, + public: &str, + ) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .team_named(team, public) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// `team_public_model_names_of`: the public names the team index holds for `team`. + fn team_public_names_of(&self, team: &str) -> Vec { + self.router + .registry() + .host_indexes() + .team_public_names_of(team) + } + + /// The positions of the deployments whose `litellm_params.model` is `model`. + fn litellm_model_positions(&self, model: &str) -> Vec { + self.router + .registry() + .deployments(&DeploymentFilter::default()) + .iter() + .enumerate() + .filter(|(_, deployment)| { + deployment.raw_params.get("model").and_then(Value::as_str) == Some(model) + }) + .map(|(position, _)| position) + .collect() + } + + /// `PatternMatchRouter.route` over the wildcard router (or `team`'s): the first pattern's + /// deployments, each with the model the wildcards matched, or None. + #[pyo3(signature = (team, request, filtered=None))] + fn pattern_route<'py>( + &self, + py: Python<'py>, + team: Option<&str>, + request: &str, + filtered: Option>, + ) -> PyResult>> { + let registry = self.router.registry(); + let indexes = registry.host_indexes(); + let router = match team { + None => Some(&indexes.patterns), + Some(team) => indexes + .team_patterns + .iter() + .find(|(held, _)| held == team) + .map(|(_, router)| router), + }; + router + .and_then(|router| router.route(request, filtered.as_deref())) + .map(|routed| to_py(py, &Value::Array(routed)).map(|value| value.into_bound(py))) + .transpose() + } + + /// `get_models_by_pattern`: the models the first matching (request, pattern) pair routes to. + fn models_by_pattern(&self, team: Option<&str>, requests: Vec) -> Option> { + let registry = self.router.registry(); + let indexes = registry.host_indexes(); + let router = match team { + None => Some(&indexes.patterns), + Some(team) => indexes + .team_patterns + .iter() + .find(|(held, _)| held == team) + .map(|(_, router)| router), + }; + router.and_then(|router| router.models_by_pattern(&requests)) + } + + /// The indexes the host hands out (`HostIndexes`): id map, name and team indexes, group and + /// deployment names, team public names, provider default ids and the pattern routers. + fn host_indexes<'py>(&self, py: Python<'py>) -> PyResult> { + let indexes = serde_json::to_value(self.router.registry().host_indexes()) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + to_py(py, &indexes).map(|value| value.into_bound(py)) + } + + /// `provider_default_deployment_ids` as the indexes hold it now: the ids of the wildcard + /// deployments the cooldown check skips. + fn provider_default_ids(&self) -> Vec { + self.router + .registry() + .host_indexes() + .provider_default_ids + .clone() + } + + /// The provider the host resolved when it built the deployment with this id (its + /// `provider` build step), or None when there is no such deployment or it never went + /// through a host build. + fn provider(&self, id: &str) -> Option { + self.router + .registry() + .deployment(id) + .and_then(|deployment| deployment.resolved_provider.clone()) + } + + /// What `name` resolves to: `{"model_group", "kind", "ids"}`, or None for an unknown name. + #[pyo3(signature = (name, specific_deployment=false))] + fn resolve<'py>( + &self, + py: Python<'py>, + name: &str, + specific_deployment: bool, + ) -> PyResult>> { + let registry = self.router.registry(); + let Some(resolution) = registry.resolve( + name, + ResolveScope { + specific_deployment, + }, + ) else { + return Ok(None); + }; + let dict = PyDict::new(py); + dict.set_item("model_group", resolution.group)?; + dict.set_item("kind", format!("{:?}", resolution.kind))?; + dict.set_item( + "ids", + PyList::new(py, resolution.deployments.iter().map(|d| d.id.clone()))?, + )?; + Ok(Some(dict)) + } + + /// The deployments a pick for `model` would resolve to for `request`, before any team, + /// cooldown or strategy filter (`_core_get_routable_deployments`); empty when it refuses. + fn routable<'py>( + &self, + py: Python<'py>, + model: &str, + request: &Bound<'py, PyAny>, + ) -> PyResult> { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let records = litellm_router_core::resolve::routable( + &self.router.registry(), + &self.router.settings().dynamic, + &request, + ) + .iter() + .map(|d| to_py(py, &host::deployment_record(d))) + .collect::>>()?; + PyList::new(py, records) + } + + /// `_get_routing_context` for a pick of `model`: `(strategy, scope, group)`, spelled as a + /// strategy select step spells them. A name that resolves to nothing reads its routing group + /// membership as is. + #[pyo3(signature = (model, routing_strategy=None))] + fn routing_context( + &self, + model: &str, + routing_strategy: Option<&str>, + ) -> (String, &'static str, Option) { + let registry = self.router.registry(); + let settings = self.router.settings(); + let (kind, group) = registry + .resolve(model, ResolveScope::default()) + .map_or((ResolveKind::Group, model.to_owned()), |resolution| { + (resolution.kind, resolution.group) + }); + let (strategy, scope) = routing_context( + ®istry, + &settings.routing_strategy, + routing_strategy, + &kind, + &group, + ); + let (scope, group) = scope.wire(); + (strategy.to_string(), scope, group.map(str::to_owned)) + } + + fn retire(&self) { + self.router.retire(); + } + + /// What may still take over a stream from `origin` that fails before content, and the + /// retry budget for a failure of `kind` (see `stream::policy`). + #[pyo3(signature = (request, origin, kind=None))] + fn stream_policy<'py>( + &self, + py: Python<'py>, + request: &Bound<'py, PyAny>, + origin: &Bound<'py, PyAny>, + kind: Option<&str>, + ) -> PyResult> { + stream::policy(py, &self.router, request, origin, kind) + } + + /// Retry a stream from `origin` that failed before content within its group, driven like + /// `start`: `done` when a retry returned a response, `failed` with the newest failure. + fn retry_stream( + &self, + op: &str, + request: &Bound<'_, PyAny>, + origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, + ) -> PyResult { + stream::recover( + &self.router, + stream::Recovery::Retry, + op, + request, + origin, + failure, + ) + } + + /// The fallback walk for a stream from `origin` that failed before content, driven like + /// `start`. + fn fallback_stream( + &self, + op: &str, + request: &Bound<'_, PyAny>, + origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, + ) -> PyResult { + stream::recover( + &self.router, + stream::Recovery::Fallback, + op, + request, + origin, + failure, + ) + } + + /// `get_num_retries_from_retry_policy` for a failure shaped like an attempt outcome. + #[pyo3(signature = (failure, model_group=None))] + fn retry_policy_retries( + &self, + failure: &Bound<'_, PyDict>, + model_group: Option<&str>, + ) -> PyResult> { + retries::policy_retries(&self.router, failure, model_group) + } + + /// `_core_get_allowed_fails_from_policy` for a failure shaped like an attempt outcome. + fn allowed_fails_for(&self, failure: &Bound<'_, PyDict>) -> PyResult> { + retries::allowed_fails(&self.router, failure) + } + + /// `should_retry_this_error` as a predicate, for a failure shaped like an attempt outcome. + fn should_retry( + &self, + failure: &Bound<'_, PyDict>, + counts: (usize, usize), + fallbacks: (bool, bool, bool), + ) -> PyResult { + retries::should_retry(failure, counts, fallbacks) + } +} + +/// One router call, driven a step at a time by Python. +#[pyclass(frozen)] +pub(crate) struct RouterCall { + driver: Mutex, +} + +impl RouterCall { + fn new( + future: impl std::future::Future> + Send + 'static, + ) -> Self { + Self { + driver: Mutex::new(Driver::new(future)), + } + } +} + +#[pymethods] +impl RouterCall { + /// Hand back the outcome of the last step (`None` on the first call) and return the next + /// step, or the call's end. + #[pyo3(signature = (outcome))] + fn next<'py>( + &self, + py: Python<'py>, + outcome: Option>, + ) -> PyResult> { + let mut driver = self.driver.lock().unwrap_or_else(PoisonError::into_inner); + let response = match driver.pending_call() { + None => match outcome { + None => None, + Some(_) => return Err(drive_error_to_pyerr(DriveError::UnexpectedResponse)), + }, + Some(call) => Some(host_call_response(call, outcome)?), + }; + let step = driver.resume(response).map_err(drive_error_to_pyerr)?; + let dict = step_dict(py, step)?; + dict.set_item("events", events_list(py, driver.take_events())?)?; + Ok(dict) + } +} + +/// `(group, names)` pairs, in the order the groups appear. +type AccessGroups = Vec<(String, Vec)>; + +/// Lookup rows as the host reads them. +fn rows_to_py<'py>(py: Python<'py>, rows: &[Row]) -> PyResult> { + let rows = Value::Array(rows.iter().map(Row::wire).collect()); + to_py(py, &rows).map(|value| value.into_bound(py)) +} + +fn row_to_py<'py>(py: Python<'py>, row: &Row) -> PyResult> { + to_py(py, &row.wire()).map(|value| value.into_bound(py)) +} + +/// `{"aliases": [{alias, target}], "groups": [{name, members}], "wildcard": [requests]}`. +fn list_scope(scope: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(document(scope)?) + .map_err(|error| PyValueError::new_err(format!("bad lookup scope: {error}"))) +} + +/// Read Python's outcome for `call`. +fn host_call_response( + call: &HostCall, + outcome: Option>, +) -> PyResult { + let missing = || { + drive_error_to_pyerr(DriveError::MissingResponse { + expected: call.kind(), + }) + }; + // `{"raised": True, ...}`: the step's work raised, reported as an attempt's failure. + if let HostCall::Select { .. } | HostCall::PreRoute { .. } | HostCall::AllowFallback { .. } = + call + && let Some(failure) = outcome.as_ref().and_then(|o| o.cast::().ok()) + && failure.contains("raised")? + { + return match attempt_outcome(failure)? { + Err(error) => Ok(HostCallResponse::Raised(error)), + Ok(_) => Err(PyValueError::new_err("a `raised` outcome is a failure")), + }; + } + match call { + HostCall::Sleep { .. } => Ok(HostCallResponse::Slept), + HostCall::AllowFallback { .. } => Ok(HostCallResponse::AllowFallback( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::RefuseFallbacks { .. } => Ok(HostCallResponse::RefuseFallbacks( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::Select { + stage: Stage::PreCall, + .. + } => { + let facts = from_py_argument(&outcome.ok_or_else(missing)?)?; + serde_json::from_value(facts) + .map(HostCallResponse::PreCall) + .map_err(|error| PyValueError::new_err(format!("bad pre-call facts: {error}"))) + } + HostCall::Select { + stage: Stage::RoutingPlugins { .. } | Stage::ProviderRequest, + .. + } => { + // `{"models": [...] | None}`: a bare None would read as no outcome. + let outcome = outcome.ok_or_else(missing)?; + let models = outcome.cast::()?.get_item("models")?; + Ok(HostCallResponse::Models(match models { + Some(models) if !models.is_none() => Some(models.extract()?), + _ => None, + })) + } + HostCall::Select { + stage: Stage::CallbackFilters { .. }, + .. + } => { + // The remaining ids, or a failure mapping as an attempt reports one. + let outcome = outcome.ok_or_else(missing)?; + if let Ok(failure) = outcome.cast::() { + return match attempt_outcome(failure)? { + Err(error) => Ok(HostCallResponse::Filtered(Err(error))), + Ok(_) => Err(PyValueError::new_err( + "a `callback_filters` outcome is the remaining ids or a failure", + )), + }; + } + Ok(HostCallResponse::Filtered(Ok(outcome.extract()?))) + } + HostCall::Select { .. } => Ok(HostCallResponse::Select( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::PreRoute { .. } => pre_route_outcome(outcome).map(HostCallResponse::PreRoute), + HostCall::Attempt { .. } => { + let outcome = outcome.ok_or_else(missing)?; + let outcome = outcome.cast::()?; + let blocked = match outcome.get_item("blocked")? { + Some(value) if !value.is_none() => { + let name: String = value.extract()?; + Some(name.parse::().map_err(|_| { + PyValueError::new_err(format!( + "`{name}` is not a way a response is blocked" + )) + })?) + } + Some(_) | None => None, + }; + let deferred = outcome + .get_item("deferred")? + .is_some_and(|flag| flag.is_truthy().unwrap_or(false)); + let tracks_usage = outcome + .get_item("tracks_usage")? + .map(|flag| flag.extract::>()) + .transpose()? + .flatten(); + attempt_outcome(outcome).map(|result| match result { + Ok(total_tokens) if blocked.is_some() || deferred || tracks_usage.is_some() => { + HostCallResponse::Served { + total_tokens, + blocked, + deferred, + tracks_usage, + } + } + result => HostCallResponse::Attempt(result), + }) + } + HostCall::Cache { op, .. } => cache_outcome(*op, outcome).map(HostCallResponse::Cache), + HostCall::Secrets { .. } => Ok(HostCallResponse::Secrets(from_py_argument( + &outcome.ok_or_else(missing)?, + )?)), + // `None` is a response here (no row, no entry), so a missing outcome reads as one. + HostCall::ModelInfo { .. } => Ok(HostCallResponse::ModelInfo(match outcome { + Some(outcome) if !outcome.is_none() => from_py_argument(&outcome)?, + _ => Value::Null, + })), + HostCall::Build { kind, .. } => { + let outcome = outcome.ok_or_else(missing)?; + let outcome = outcome.cast::()?; + let invalid = outcome + .get_item("invalid")? + .is_some_and(|flag| flag.is_truthy().unwrap_or(false)); + match kind { + StepKind::Provider => Ok(HostCallResponse::Provided(if invalid { + Provided::Invalid + } else { + Provided::Provider(match outcome.get_item("provider")? { + Some(provider) if !provider.is_none() => Some(provider.extract()?), + _ => None, + }) + })), + _ => { + let built = if invalid { + Built::Invalid + } else { + match outcome.get_item("entry")? { + Some(entry) if !entry.is_none() => { + Built::Entry(from_py_argument(&entry)?) + } + _ => Built::Skip, + } + }; + Ok(if *kind == StepKind::Validate { + HostCallResponse::Validated(built) + } else { + HostCallResponse::Added(built) + }) + } + } + } + } +} + +/// A pre-route response: `None`, a model group name, or a custom strategy's deployment document. +fn pre_route_outcome(outcome: Option>) -> PyResult> { + let Some(outcome) = outcome.filter(|outcome| !outcome.is_none()) else { + return Ok(None); + }; + if let Ok(model) = outcome.cast::() { + return Ok(Some(PreRouted::Model(model.to_str()?.to_owned()))); + } + let spec: DeploymentSpec = serde_json::from_value(document(&outcome)?) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let deployment = Deployment::from_spec(spec) + .map_err(|error| PyValueError::new_err(Report::new(&error).to_string()))?; + Ok(Some(PreRouted::Deployment(Arc::new(deployment)))) +} + +/// `get`: the JSON value or None; `batch_get`: a list of JSON values or None, one per key; +/// `increment`: the new int or float; `set` and `delete`: None. +fn cache_outcome(op: CacheOp, outcome: Option>) -> PyResult { + let outcome = outcome.filter(|outcome| !outcome.is_none()); + let json = |value: Option| value.filter(|value| !value.is_null()); + match op { + CacheOp::Get => outcome + .map(|outcome| from_py_argument::(&outcome)) + .transpose() + .map(|value| CacheOutcome::Value(json(value))), + CacheOp::BatchGet => { + let outcome = outcome.ok_or_else(|| { + PyValueError::new_err("a `batch_get` cache outcome must be a list, not None") + })?; + if !outcome.is_instance_of::() { + return Err(PyValueError::new_err( + "a `batch_get` cache outcome must be a list with one entry per key", + )); + } + let values: Vec> = from_py_argument(&outcome)?; + Ok(CacheOutcome::Values(values.into_iter().map(json).collect())) + } + CacheOp::Increment => outcome + .filter(|outcome| !outcome.is_instance_of::()) + .and_then(|outcome| outcome.extract::().ok()) + .map(CacheOutcome::Number) + .ok_or_else(|| { + PyValueError::new_err("an `increment` cache outcome must be the new int or float") + }), + CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage => match outcome { + None => Ok(CacheOutcome::Stored), + Some(_) => Err(PyValueError::new_err(format!( + "a `{op}` cache step has no outcome, so pass None" + ))), + }, + } +} + +/// `{"ok": True, "total_tokens": int}` (an attempt's may add `"blocked": str`, `"deferred": bool` +/// and `"tracks_usage": bool`) or `{"ok": False, "kind": str, "status": int | None, +/// "retry_after": float | None, "num_retries": int | None, "detail": str | None, +/// "outside_deployment": bool, "caller_scoped": bool, "status_absent": bool, +/// "uncounted_request": bool}`. +pub(crate) fn attempt_outcome(outcome: &Bound<'_, PyDict>) -> PyResult> { + fn item<'py, T: FromPyObjectOwned<'py>>( + outcome: &Bound<'py, PyDict>, + key: &str, + ) -> PyResult> { + match outcome.get_item(key)? { + Some(value) if !value.is_none() => Ok(Some(value.extract().map_err(Into::into)?)), + Some(_) | None => Ok(None), + } + } + let ok: bool = item(outcome, "ok")? + .ok_or_else(|| PyValueError::new_err("an attempt outcome needs `ok`"))?; + if ok { + return Ok(Ok(item(outcome, "total_tokens")?.unwrap_or(0))); + } + let kind: String = item(outcome, "kind")? + .ok_or_else(|| PyValueError::new_err("a failed attempt outcome needs `kind`"))?; + let kind: FailureKind = kind + .parse() + .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind")))?; + let retry_after = item::(outcome, "retry_after")? + .map(|seconds| { + Duration::try_from_secs_f64(seconds) + .map_err(|_| PyValueError::new_err("`retry_after` must be a non-negative number")) + }) + .transpose()?; + let mut failure = AttemptError::new(kind, item(outcome, "status")?); + failure.retry_after = retry_after; + failure.num_retries = item(outcome, "num_retries")?; + failure.failed_deployment_id = item(outcome, "failed_deployment_id")?; + failure.detail = item::(outcome, "detail")?.map(Into::into); + failure.recorded = item::(outcome, "recorded")?.unwrap_or(false); + failure.names_provider = item::(outcome, "llm_provider")?.unwrap_or(false); + failure.agentic_followup = item::(outcome, "agentic_followup")?.unwrap_or(false); + failure.outside_deployment = item::(outcome, "outside_deployment")?.unwrap_or(false); + failure.status_absent = item::(outcome, "status_absent")?.unwrap_or(false); + failure.uncounted_request = item::(outcome, "uncounted_request")?.unwrap_or(false); + failure.caller_scoped = item::(outcome, "caller_scoped")?.unwrap_or(false); + failure.classes = item::>(outcome, "classes")? + .unwrap_or_default() + .into_iter() + .map(Into::into) + .collect(); + Ok(Err(failure)) +} + +/// `[{"kind": "retries_exhausted", "model_group": str, "attempted": int, "max": int} +/// | {"kind": "fallback", "to": str, "order": int | None, "level": int}]`. +/// The `cause` of a `retry_logged` or `retry_policy_raised` event, with what names it. +fn cause_items(dict: &Bound<'_, PyDict>, cause: RetryCause) -> PyResult<()> { + match cause { + RetryCause::Attempt => dict.set_item("cause", "attempt"), + RetryCause::Level => dict.set_item("cause", "level"), + RetryCause::Other => dict.set_item("cause", "other"), + RetryCause::Pick(picked) => { + dict.set_item("cause", "pick")?; + pick_items(dict, &picked.0) + } + } +} + +/// A pick's error, as the `failed` step names it (`error_kind` and what names it), with the +/// model its message names (`failed_model`) when the error carries one. +fn pick_items(dict: &Bound<'_, PyDict>, error: &Error) -> PyResult<()> { + error_items(dict, error)?; + match error { + Error::NoHealthyDeployments { model } | Error::UnknownModel { model } => { + dict.set_item("failed_model", model) + } + _ => Ok(()), + } +} + +/// `error_kind` and the fields that name the error. +fn error_items(dict: &Bound<'_, PyDict>, error: &Error) -> PyResult<()> { + dict.set_item("error", Report::new(error).to_string())?; + dict.set_item("error_kind", <&'static str>::from(ErrorKind::from(error)))?; + match error { + Error::EndpointNotSupported { model, message } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", message) + } + Error::OnlyMarkers { model } + | Error::UnknownModel { model } + | Error::NoPassThroughDeployments { model } + | Error::PtuReserved { model } + | Error::MultipleTeams { model } + | Error::DefaultDeploymentRaises { model } + | Error::IndexOutOfRange { model } => dict.set_item("error_model", model), + Error::ContextWindowExceeded { model, detail } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", detail) + } + Error::UnsupportedParams { model, params } => { + dict.set_item("error_model", model)?; + dict.set_item("error_params", PyList::new(dict.py(), params)?) + } + Error::RoutingPluginsLeftNothing { models } => { + dict.set_item("error_params", PyList::new(dict.py(), models)?) + } + Error::Reserved { + model, + until, + timezone, + } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", format!("{until} {timezone}")) + } + Error::NonNumericWeight { metric } => dict.set_item("error_detail", metric), + Error::MissingRoutingGroup { group } => dict.set_item("error_model", group), + Error::ModelInfo { exception, detail } => { + dict.set_item("error_exception", *exception)?; + dict.set_item("error_detail", detail) + } + _ => Ok(()), + } +} + +fn events_list(py: Python<'_>, events: Vec) -> PyResult> { + let list = PyList::empty(py); + for event in events { + let dict = PyDict::new(py); + match event { + RouterEvent::RetriesExhausted { + model_group, + attempted, + max, + level, + picked, + } => { + dict.set_item("kind", "retries_exhausted")?; + dict.set_item("model_group", model_group)?; + dict.set_item("attempted", attempted)?; + dict.set_item("max", max)?; + dict.set_item("level", level)?; + dict.set_item("picked", picked)?; + } + RouterEvent::Retry { retry, .. } => { + dict.set_item("kind", "retry")?; + dict.set_item("retry", retry)?; + } + RouterEvent::FallingBack { level, model } => { + dict.set_item("kind", "falling_back")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + } + RouterEvent::OrderChainFailed { level, last_hop } => { + dict.set_item("kind", "order_chain_failed")?; + dict.set_item("level", level)?; + dict.set_item("last_hop", last_hop)?; + } + RouterEvent::FallbacksFailed { + level, + chain, + last_hop, + } => { + dict.set_item("kind", "fallbacks_failed")?; + dict.set_item("level", level)?; + dict.set_item("chain", chain)?; + dict.set_item("last_hop", last_hop)?; + } + RouterEvent::Fallback { + from, + to, + order, + depth, + level, + scope, + parent, + max_fallbacks, + unroutable, + } => { + dict.set_item("kind", "fallback")?; + dict.set_item("from", from)?; + dict.set_item("to", to)?; + dict.set_item("order", order)?; + dict.set_item("depth", depth)?; + dict.set_item("level", level)?; + dict.set_item("scope", scope)?; + dict.set_item("parent", parent)?; + dict.set_item("max_fallbacks", max_fallbacks)?; + if let Some(model) = unroutable { + dict.set_item("unroutable", to_py(py, &model)?)?; + } + } + RouterEvent::Blocked { + level, + model, + blocked, + } => { + dict.set_item("kind", "blocked")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("blocked", <&'static str>::from(blocked))?; + } + RouterEvent::FallbackDone { + from, + to, + level, + scope, + parent, + succeeded, + } => { + dict.set_item("kind", "fallback_done")?; + dict.set_item("from", from)?; + dict.set_item("to", to)?; + dict.set_item("level", level)?; + dict.set_item("scope", scope)?; + dict.set_item("parent", parent)?; + dict.set_item("succeeded", succeeded)?; + } + RouterEvent::RetryLogged { + scope, + level, + model, + attempted_retries, + cause, + } => { + dict.set_item("kind", "retry_logged")?; + dict.set_item("scope", scope)?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("attempted_retries", attempted_retries)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetryPolicyRaised { + level, + model, + cause, + } => { + dict.set_item("kind", "retry_policy_raised")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetryCountRaised { + level, + model, + cause, + } => { + dict.set_item("kind", "retry_count_raised")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetriesOpened { + scope, + max_retries, + model_group_size, + } => { + dict.set_item("kind", "retries_opened")?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + dict.set_item("model_group_size", model_group_size)?; + } + RouterEvent::RetryBudget { scope, max_retries } => { + dict.set_item("kind", "retry_budget")?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + } + RouterEvent::PickFailed { + level, + model, + error, + } => { + dict.set_item("kind", "pick_failed")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + pick_items(&dict, &error.0)?; + } + RouterEvent::LevelFailed { + level, + model, + error, + handling, + } => { + dict.set_item("kind", "level_failed")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("handling", handling)?; + pick_items(&dict, &error.0)?; + } + RouterEvent::MockFailed { + mock, + level, + model, + num_retries, + } => { + dict.set_item("kind", "mock_failed")?; + dict.set_item("mock", <&'static str>::from(mock))?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("num_retries", num_retries)?; + } + RouterEvent::CooldownSet { + deployment_id, + cooldown, + } => { + dict.set_item("kind", "cooldown_set")?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("cooldown_time", cooldown.as_secs_f64())?; + } + RouterEvent::Selected { + deployment_id, + pick, + .. + } => { + dict.set_item("kind", "selected")?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("pick_seconds", pick.as_secs_f64())?; + } + _ => continue, + } + list.append(dict)?; + } + Ok(list) +} + +fn step_dict(py: Python<'_>, step: Step) -> PyResult> { + let dict = PyDict::new(py); + let kind = match &step { + Step::HostCall(call) => call.kind(), + Step::Done(_) => StepKind::Done, + Step::Failed(_) => StepKind::Failed, + }; + dict.set_item("kind", <&'static str>::from(kind))?; + match step { + Step::HostCall(call) => host_call_items(py, &dict, call)?, + Step::Done(finish) => done_items(&dict, finish)?, + Step::Failed(error) => { + let attempt = error.attempt(); + error_items(&dict, &error)?; + dict.set_item("attempt_failed", attempt.is_some())?; + dict.set_item("status", attempt.and_then(|failure| failure.status))?; + dict.set_item( + "failure_kind", + attempt.map(|failure| <&'static str>::from(failure.kind)), + )?; + } + } + Ok(dict) +} + +fn host_call_items(py: Python<'_>, dict: &Bound<'_, PyDict>, call: HostCall) -> PyResult<()> { + match call { + HostCall::Attempt { + deployment_id, + deployment, + model, + model_group, + target_order, + candidate_ids, + retry, + fallback_depth, + fallback_level, + timeout, + params, + scope, + max_retries, + model_group_size, + } => { + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + dict.set_item("model_group_size", model_group_size)?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("deployment", to_py(py, &deployment)?)?; + dict.set_item("model", model)?; + dict.set_item("model_group", model_group)?; + dict.set_item("target_order", target_order)?; + dict.set_item("candidate_ids", PyList::new(py, candidate_ids)?)?; + dict.set_item("retry", retry)?; + dict.set_item("fallback_depth", fallback_depth)?; + dict.set_item("fallback_level", fallback_level)?; + dict.set_item("timeout", timeout)?; + dict.set_item("params", params.map(|p| to_py(py, &p)).transpose()?) + } + HostCall::Sleep { seconds } => dict.set_item("seconds", seconds), + HostCall::AllowFallback { target } => dict.set_item("target", target), + HostCall::RefuseFallbacks { level, targets } => { + dict.set_item("level", level)?; + let targets = targets.0.iter().cloned().map(Value::Object).collect(); + dict.set_item("targets", to_py(py, &Value::Array(targets))?) + } + HostCall::Select { + stage, + model_group, + name, + scope, + group, + pool, + candidates, + } => { + let (stage, strategy) = match stage { + Stage::Plugins => ("plugins", None), + Stage::Strategy(strategy) => ("strategy", Some(strategy)), + Stage::Tags => ("tags", None), + Stage::PreCall => ("pre_call", None), + Stage::TextOnly => ("text_only", None), + Stage::RouteIncapable { handler } => { + dict.set_item("handler", handler)?; + ("route_incapable", None) + } + Stage::MediaRefusing => ("media_refusing", None), + Stage::ProviderRequest => ("provider_request", None), + Stage::CallbackFilters { + excluded_ids, + skipped_ids, + target_order, + } => { + dict.set_item("excluded_ids", excluded_ids)?; + dict.set_item("skipped_ids", skipped_ids)?; + dict.set_item("target_order", target_order)?; + ("callback_filters", None) + } + Stage::RoutingPlugins { run } => { + dict.set_item("run", run)?; + ("routing_plugins", None) + } + }; + dict.set_item("pool", PyList::new(py, pool)?)?; + dict.set_item("stage", stage)?; + dict.set_item("strategy", strategy)?; + dict.set_item("model_group", model_group)?; + dict.set_item("model", name)?; + dict.set_item("scope", scope)?; + dict.set_item("group", group)?; + dict.set_item("candidates", PyList::new(py, candidates)?) + } + HostCall::PreRoute { + model, + fallback_depth, + retry, + } => { + dict.set_item("model", model)?; + dict.set_item("fallback_depth", fallback_depth)?; + dict.set_item("retry", retry) + } + HostCall::Cache { + op, + key, + keys, + value, + ttl, + local, + } => { + dict.set_item("op", <&'static str>::from(op))?; + dict.set_item("key", key)?; + dict.set_item("keys", keys.map(|keys| PyList::new(py, keys)).transpose()?)?; + dict.set_item("value", value.map(|value| to_py(py, &value)).transpose()?)?; + dict.set_item("ttl", ttl)?; + dict.set_item("local", local) + } + HostCall::Secrets { index, refs } => { + dict.set_item("index", index)?; + let refs = refs + .into_iter() + .map(|secret| { + let item = PyDict::new(py); + item.set_item("name", secret.name)?; + match secret.at { + SecretAt::Param(key) => { + item.set_item("key", key)?; + item.set_item("env_only", false)?; + } + SecretAt::DataSource { source, key } => { + item.set_item("source", source)?; + item.set_item("key", key)?; + item.set_item("env_only", true)?; + } + } + Ok(item) + }) + .collect::>>()?; + dict.set_item("refs", PyList::new(py, refs)?) + } + HostCall::Build { + kind: _, + path, + index, + entry, + declared_id, + duplicate_id, + provider, + } => { + dict.set_item("path", <&'static str>::from(path))?; + dict.set_item("index", index)?; + dict.set_item("entry", to_py(py, &entry)?)?; + dict.set_item("declared_id", declared_id)?; + dict.set_item("duplicate_id", duplicate_id)?; + dict.set_item("provider", provider) + } + HostCall::ModelInfo { lookup } => { + if let Value::Object(lookup) = lookup { + for (key, value) in lookup { + dict.set_item(key, to_py(py, &value)?)?; + } + } + Ok(()) + } + } +} + +fn done_items(dict: &Bound<'_, PyDict>, finish: Finish) -> PyResult<()> { + match finish { + Finish::Call(response) => { + dict.set_item("deployment_id", response.deployment_id)?; + dict.set_item("model_group", response.model_group)?; + dict.set_item("attempted_retries", response.attempted_retries)?; + dict.set_item("max_retries", response.max_retries)?; + dict.set_item("attempted_fallbacks", response.attempted_fallbacks)?; + dict.set_item("total_tokens", response.output.total_tokens) + } + Finish::Selected { + deployment_id, + model_group, + deployment, + } => { + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("model_group", model_group)?; + dict.set_item("deployment", to_py(dict.py(), &deployment)?) + } + Finish::Reported { cooldown } => { + dict.set_item("cooled", cooldown.is_some())?; + dict.set_item("cooldown_time", cooldown.map(|c| c.as_secs_f64())) + } + Finish::Built(built) => { + let py = dict.py(); + let records = built + .into_iter() + .map(|deployment| { + let item = PyDict::new(py); + item.set_item("index", deployment.index)?; + item.set_item( + "id", + deployment.spec.model_info.get("id").and_then(Value::as_str), + )?; + item.set_item("provider", deployment.spec.provider)?; + Ok(item) + }) + .collect::>>()?; + dict.set_item("built", PyList::new(py, records)?) + } + Finish::Counted { key } => dict.set_item("key", key), + Finish::RouterModelInfo(merged) => { + let py = dict.py(); + match merged { + Some(merged) => { + dict.set_item("info", to_py(py, &Value::Object(merged.info))?)?; + dict.set_item("user_keys", merged.user_keys) + } + None => dict.set_item("info", py.None()), + } + } + Finish::DeploymentModelInfo(info) => { + let py = dict.py(); + match info { + Some(info) => dict.set_item("info", to_py(py, &Value::Object(info))?), + None => dict.set_item("info", py.None()), + } + } + Finish::Value(value) => dict.set_item("value", to_py(dict.py(), &value)?), + Finish::Healthy { + single, + deployment_ids, + deployments, + } => { + dict.set_item("single", single)?; + dict.set_item("deployment_ids", deployment_ids)?; + let records = deployments + .iter() + .map(|d| to_py(dict.py(), d)) + .collect::>>()?; + dict.set_item("deployments", PyList::new(dict.py(), records)?) + } + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ffi::c_str, prelude::*, types::PyDict}; + use rstest::rstest; + + const CONFIG: &str = r#"{"model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}} + ]}"#; + + const PAIR: &str = r#"{"model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}} + ]}"#; + + #[rstest] + fn python_drives_calls_and_selections_through_the_handle() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("native", crate::native_module(py)).unwrap(); + locals.set_item("config", CONFIG).unwrap(); + locals.set_item("pair", PAIR).unwrap(); + py.run( + c_str!( + r#" +import time +store = {} +asked = [] + +def answer_cache(step): + asked.append(step) + op, key = step["op"], step["key"] + if op == "get": + return store.get(key) + if op == "batch_get": + return [store.get(k) for k in step["keys"]] + if op == "set": + store[key] = step["value"] + return None + if op == "increment": + store[key] = store.get(key, 0) + step["value"] + return store[key] + if op == "delete": + store.pop(key, None) + return None + raise AssertionError(op) + +def drive(call, outcome): + step = call.next(outcome) + while step["kind"] == "cache": + step = call.next(answer_cache(step)) + return step + +handle = native.RouterCoreHandle(config, {"num_retries": 1}) +picked = drive(handle.select("primary", {}), None) +assert (picked["kind"], picked["deployment_id"], picked["model_group"]) == ("done", "a1", "primary"), picked +assert [step["op"] for step in asked] == ["batch_get"] +assert list(asked[0]) == ["kind", "op", "key", "keys", "value", "ttl", "local", "events"], list(asked[0]) +call = handle.start("acompletion", '{"model": "primary"}') +version = handle.registry_version() +assert handle.apply({"delete": ["gone"]}) == version + 1 +steps = [drive(call, None)] +steps.append(drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})) +steps.append(drive(call, None)) +steps.append(drive(call, {"ok": True, "total_tokens": 5})) +kinds = [step["kind"] for step in steps] + +cooled = native.RouterCoreHandle(pair, {"num_retries": 0}) +call = cooled.start("acompletion", {"model": "primary"}) +attempt = drive(call, None) +tried = attempt["deployment_id"] +assert attempt["deployment"]["model_info"]["id"] == tried, attempt +assert attempt["deployment"]["litellm_params"]["model"].startswith("openai/"), attempt +assert [d["model_info"]["id"] for d in cooled.deployments()] == ["a1", "a2"] +assert cooled.deployment("a2")["model_name"] == "primary" +assert cooled.resolve("primary")["ids"] == ["a1", "a2"] +assert cooled.resolve("nope") is None +assert drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})["kind"] == "failed" +# The core records the failure itself: the 429 cools the deployment down in the host's store. +assert f"deployment:{tried}:cooldown" in store, store +other = drive(cooled.select("primary", {}), None) +assert other["kind"] == "done" and other["deployment_id"] != tried, other + +call = handle.select("primary", {}) +step = call.next(None) +assert step["op"] == "batch_get" +for bad in ([], [None, None], "nope", None): + try: + call.next(bad) + except ValueError: + pass + else: + raise AssertionError(f"{bad!r} should not answer a batch_get of one key") +assert drive(call, [None])["kind"] == "done" +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let kinds: Vec = locals + .get_item("kinds") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(kinds, ["attempt", "sleep", "attempt", "done"]); + }); + } + + #[rstest] + #[case::unknown_key(r#"{"routing_strategy_args": {}}"#, true)] + #[case::bad_json("{", false)] + #[case::not_an_object("[]", false)] + fn construction_failures_raise_the_class_python_reads( + #[case] config: &str, + #[case] declined: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let error = crate::native_module(py) + .getattr("RouterCoreHandle") + .and_then(|handle| handle.call1((config, "{}"))) + .unwrap_err(); + assert_eq!( + error.is_instance_of::(py), + declined, + "{error}" + ); + assert_eq!( + error.is_instance_of::(py), + !declined, + "{error}" + ); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/retries.rs b/litellm-rust/crates/python-bridge/src/router/retries.rs new file mode 100644 index 00000000000..949aae96b34 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/retries.rs @@ -0,0 +1,55 @@ +//! The retry decisions `RouterCoreHandle` exposes as members: `get_num_retries_from_retry_policy` +//! and `should_retry_this_error`, decided by router-core for a failure Python describes the way it +//! describes a failed attempt. + +use litellm_router_core::{ + AttemptError, Router, + retry::{RetryContext, resolve_retry_policy, should_retry as core_should_retry}, +}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; + +fn failure(outcome: &Bound<'_, PyDict>) -> PyResult { + match super::attempt_outcome(outcome)? { + Err(failure) => Ok(failure), + Ok(_) => Err(PyValueError::new_err("a failure needs `ok: False`")), + } +} + +pub(crate) fn policy_retries( + router: &Router, + outcome: &Bound<'_, PyDict>, + model_group: Option<&str>, +) -> PyResult> { + let failure = failure(outcome)?; + let settings = router.settings(); + let policy = match model_group { + Some(group) => resolve_retry_policy( + settings.retry_policy.as_ref(), + group, + Some(&settings.model_group_retry_policy), + ), + None => settings.retry_policy.as_ref(), + }; + Ok(policy.and_then(|policy| policy.retries_for(&failure))) +} + +pub(crate) fn allowed_fails(router: &Router, outcome: &Bound<'_, PyDict>) -> PyResult> { + Ok(router.settings().allowed_fails_for(&failure(outcome)?)) +} + +pub(crate) fn should_retry( + outcome: &Bound<'_, PyDict>, + (healthy, all): (usize, usize), + (context_window, content_policy, regular): (bool, bool, bool), +) -> PyResult { + Ok(core_should_retry( + &failure(outcome)?, + RetryContext { + healthy, + all, + has_context_window_fallbacks: context_window, + has_content_policy_fallbacks: content_policy, + has_regular_fallbacks: regular, + }, + )) +} diff --git a/litellm-rust/crates/python-bridge/src/router/stream.rs b/litellm-rust/crates/python-bridge/src/router/stream.rs new file mode 100644 index 00000000000..f85d73e0778 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/stream.rs @@ -0,0 +1,94 @@ +//! The stream-recovery entry points of `RouterCoreHandle`: what may still take over a stream that +//! failed before content, the same-group stream retry, and the fallback walk re-entered from a +//! stream. The host's stream wrappers call them; the calls are driven like `start`. + +use litellm_router_core::{ + FailureKind, Op, Router, + call::{StreamOrigin, StreamPolicy}, +}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; + +use super::{RouterCall, attempt_outcome, config, config_error_to_pyerr, document, host::Finish}; + +/// A stream recovery a host wrapper asks for. +#[derive(Clone, Copy)] +pub(super) enum Recovery { + /// `_astream_retry_same_group`. + Retry, + /// The fallback walk re-entered from the stream wrapper. + Fallback, +} + +fn origin(value: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(document(value)?) + .map_err(|error| PyValueError::new_err(format!("a stream origin: {error}"))) +} + +fn kind(kind: Option<&str>) -> PyResult> { + kind.map(|kind| { + kind.parse() + .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind"))) + }) + .transpose() +} + +/// `{"can_retry", "can_fall_back", "refusal_fallback", "retry_budget": int | None, +/// "policy_applies": bool}`. +pub(super) fn policy<'py>( + py: Python<'py>, + router: &Router, + request: &Bound<'py, PyAny>, + stream_origin: &Bound<'py, PyAny>, + failure_kind: Option<&str>, +) -> PyResult> { + let request = config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let StreamPolicy { + can_retry, + can_fall_back, + refusal_fallback, + retry_budget, + } = router.stream_policy(&request, &origin(stream_origin)?, kind(failure_kind)?); + let dict = PyDict::new(py); + dict.set_item("can_retry", can_retry)?; + dict.set_item("can_fall_back", can_fall_back)?; + dict.set_item("refusal_fallback", refusal_fallback)?; + dict.set_item("retry_budget", retry_budget.map(|(budget, _)| budget))?; + dict.set_item( + "policy_applies", + retry_budget.is_some_and(|(_, applies)| applies), + )?; + Ok(dict) +} + +/// A recovery call for a stream from `stream_origin` that failed with `failure`. +pub(super) fn recover( + router: &Router, + recovery: Recovery, + op: &str, + request: &Bound<'_, PyAny>, + stream_origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, +) -> PyResult { + let op = Op::new(op); + let request = config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let stream_origin = origin(stream_origin)?; + let Err(failure) = attempt_outcome(failure)? else { + return Err(PyValueError::new_err("a stream failure needs `ok: False`")); + }; + let router = router.clone(); + Ok(RouterCall::new(async move { + let response = match recovery { + Recovery::Retry => { + router + .retry_stream(op, request, stream_origin, failure) + .await + } + Recovery::Fallback => { + router + .fallback_stream(op, request, stream_origin, failure) + .await + } + }; + response.map(Finish::Call) + })) +} diff --git a/litellm-rust/crates/router-core/Cargo.toml b/litellm-rust/crates/router-core/Cargo.toml index af97bd52c57..ff5e2e8161f 100644 --- a/litellm-rust/crates/router-core/Cargo.toml +++ b/litellm-rust/crates/router-core/Cargo.toml @@ -7,3 +7,21 @@ repository.workspace = true description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop" [dependencies] +arc-swap.workspace = true +async-trait.workspace = true +litellm-cache.workspace = true +litellm-router-types.workspace = true +jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-zoneinfo", "tzdb-bundle-always"] } +parking_lot.workspace = true +rand.workspace = true +serde.workspace = true +serde_with.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +sha2.workspace = true +strum.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/router-core/src/access.rs b/litellm-rust/crates/router-core/src/access.rs new file mode 100644 index 00000000000..1b91fa417da --- /dev/null +++ b/litellm-rust/crates/router-core/src/access.rs @@ -0,0 +1,236 @@ +//! Who may reach a deployment: model access windows (`litellm_core_utils/access_windows.py`, +//! a deployment reserved for some teams during a daily wall-clock window) and model access +//! groups (`_filter_deployments_by_model_access_groups`, a caller allowed only through access +//! groups is kept to its groups' deployments). + +use std::{collections::BTreeSet, sync::Arc}; + +use jiff::{Timestamp, tz::TimeZone}; +use serde_json::Value; + +use crate::registry::Deployment; + +/// One `model_info.access_windows` entry (`ModelAccessWindow`), validated when the deployment +/// was added. +#[derive(Clone, Debug, PartialEq)] +pub struct AccessWindow { + /// Microseconds after midnight. + pub start: u64, + pub end: u64, + pub timezone: String, + pub team_ids: Vec, +} + +const MICROS_PER_SECOND: u64 = 1_000_000; + +/// A `datetime.time` as pydantic reads it: `HH:MM[:SS[.ffffff]]`, or seconds after midnight. +fn wall_clock(value: &Value) -> Option { + match value { + Value::Number(seconds) => { + let seconds = seconds.as_f64()?; + (0.0..86_400.0) + .contains(&seconds) + .then(|| (seconds * MICROS_PER_SECOND as f64).round() as u64) + } + Value::String(text) => { + let mut parts = text.splitn(3, ':'); + let hour: u64 = parts.next()?.parse().ok()?; + let minute: u64 = parts.next()?.parse().ok()?; + let (second, micros) = match parts.next() { + None => (0, 0), + Some(rest) => { + let (whole, fraction) = rest.split_once('.').unwrap_or((rest, "")); + let digits: String = fraction.chars().take(6).collect(); + let micros = if digits.is_empty() { + 0 + } else { + format!("{digits:0<6}").parse().ok()? + }; + (whole.parse().ok()?, micros) + } + }; + (hour < 24 && minute < 60 && second < 60) + .then(|| ((hour * 60 + minute) * 60 + second) * MICROS_PER_SECOND + micros) + } + _ => None, + } +} + +impl AccessWindow { + fn parse(value: &Value) -> Option { + let window = value.as_object()?; + Some(Self { + start: wall_clock(window.get("start")?)?, + end: wall_clock(window.get("end")?)?, + timezone: window.get("timezone")?.as_str()?.to_owned(), + team_ids: window + .get("team_ids")? + .as_array()? + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + }) + } + + /// `is_window_active`: the window's wall clock contains `now`, wrapping past midnight when + /// it ends before it starts. + pub fn is_active(&self, now: Timestamp) -> bool { + let Ok(zone) = TimeZone::get(&self.timezone) else { + return false; + }; + let local = now.to_zoned(zone).time(); + let micros = ((u64::from(local.hour().unsigned_abs()) * 60 + + u64::from(local.minute().unsigned_abs())) + * 60 + + u64::from(local.second().unsigned_abs())) + * MICROS_PER_SECOND + + u64::from(local.subsec_nanosecond().unsigned_abs()) / 1_000; + if self.start < self.end { + self.start <= micros && micros < self.end + } else { + micros >= self.start || micros < self.end + } + } + + /// The window's end as `{end:%H:%M}`. + pub fn end_hhmm(&self) -> String { + let minutes = self.end / MICROS_PER_SECOND / 60; + format!("{:02}:{:02}", minutes / 60, minutes % 60) + } +} + +/// A deployment's access windows. +pub fn access_windows(deployment: &Deployment) -> Vec { + deployment + .model_info + .raw + .get("access_windows") + .and_then(Value::as_array) + .map(|windows| windows.iter().filter_map(AccessWindow::parse).collect()) + .unwrap_or_default() +} + +/// `_reservation_blocking_window`: the first active window, unless the caller's team is named +/// by one of the active windows. +fn blocking_window( + deployment: &Deployment, + team_id: Option<&str>, + now: Timestamp, +) -> Option { + let active: Vec = access_windows(deployment) + .into_iter() + .filter(|window| window.is_active(now)) + .collect(); + if let Some(team) = team_id + && active + .iter() + .any(|window| window.team_ids.iter().any(|id| id == team)) + { + return None; + } + active.into_iter().next() +} + +/// `filter_reserved_deployments`: the deployments no active window withholds from the caller, +/// and the first window that withheld one. +pub fn filter_reserved( + deployments: Vec>, + team_id: Option<&str>, + now: Timestamp, +) -> (Vec>, Option) { + let mut blocking = None; + let kept = deployments + .into_iter() + .filter( + |deployment| match blocking_window(deployment, team_id, now) { + None => true, + Some(window) => { + blocking.get_or_insert(window); + false + } + }, + ) + .collect(); + (kept, blocking) +} + +/// `_filter_deployments_by_model_access_groups`: a caller whose allowed models (`models` plus +/// `team_models`) reach `model` only through access groups is kept to the deployments in those +/// groups. `model_groups` are the access groups the rows listed for `model` carry. +pub fn filter_by_access_groups( + model: &str, + deployments: Vec>, + allowed_models: &[String], + model_groups: impl FnOnce() -> BTreeSet, +) -> Vec> { + if deployments.is_empty() || allowed_models.is_empty() { + return deployments; + } + let allowed = |name: &str| allowed_models.iter().any(|m| m == name); + if allowed(model) || allowed("*") || allowed("all-proxy-models") { + return deployments; + } + let groups: BTreeSet = model_groups() + .into_iter() + .filter(|group| allowed(group)) + .collect(); + if groups.is_empty() { + return deployments; + } + deployments + .into_iter() + .filter(|d| { + d.model_info + .access_groups + .iter() + .any(|g| groups.contains(g)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::minutes(json!("09:30"), Some(34_200_000_000))] + #[case::seconds(json!("09:30:15"), Some(34_215_000_000))] + #[case::fraction(json!("00:00:00.5"), Some(500_000))] + #[case::number(json!(60), Some(60_000_000))] + #[case::bad(json!("25:00"), None)] + fn wall_clock_reads_pydantic_times(#[case] raw: Value, #[case] expected: Option) { + assert_eq!(wall_clock(&raw), expected); + } + + #[rstest] + #[case::inside("2026-01-01T10:00:00Z", true)] + #[case::at_end("2026-01-01T17:00:00Z", false)] + #[case::before("2026-01-01T08:59:59Z", false)] + fn a_window_is_active_inside_its_wall_clock(#[case] now: &str, #[case] expected: bool) { + let window = AccessWindow { + start: wall_clock(&json!("09:00")).unwrap(), + end: wall_clock(&json!("17:00")).unwrap(), + timezone: "UTC".to_owned(), + team_ids: vec!["t".to_owned()], + }; + assert_eq!(window.is_active(now.parse().unwrap()), expected); + assert_eq!(window.end_hhmm(), "17:00"); + } + + #[test] + fn an_overnight_window_wraps_past_midnight() { + let window = AccessWindow { + start: wall_clock(&json!("22:00")).unwrap(), + end: wall_clock(&json!("02:00")).unwrap(), + timezone: "UTC".to_owned(), + team_ids: vec!["t".to_owned()], + }; + assert!(window.is_active("2026-01-01T23:00:00Z".parse().unwrap())); + assert!(window.is_active("2026-01-01T01:00:00Z".parse().unwrap())); + assert!(!window.is_active("2026-01-01T12:00:00Z".parse().unwrap())); + } +} diff --git a/litellm-rust/crates/router-core/src/build.rs b/litellm-rust/crates/router-core/src/build.rs new file mode 100644 index 00000000000..36bad012b3a --- /dev/null +++ b/litellm-rust/crates/router-core/src/build.rs @@ -0,0 +1,386 @@ +//! The deployment build: how configured entries become the registry's deployments. +//! +//! The core owns the loop (`set_model_list`, `_create_deployment` and `_add_deployment`): which +//! values are secret references, the id of a deployment that declares none, one deployment per +//! `organization`, and the order everything happens in. What needs the embedding process is a +//! host step, asked for one deployment at a time so an error surfaces exactly where the Python +//! router raises it: +//! - [`Host::secrets`]: the `os.environ/` references the entry holds; +//! - [`Host::validate`]: the host's checks, and the entry as it normalizes it; +//! - [`Host::provider`]: the provider that serves it; +//! - [`Host::added`]: the host's side effects, and the entry as the registry stores it. + +use serde_json::{Map, Value}; + +use crate::{ + error::{ConfigError, Error}, + host::{BuildAsk, BuildPath, Built, Host, Provided, SecretAt, SecretRef}, + registry::{DeploymentSpec, deployment_id}, +}; + +/// The `litellm_params` keys whose `os.environ/` value is a secret pointer the identity source +/// dereferences at use time, so the build leaves it as written. The router's copy of +/// `_WIF_POINTER_FIELDS` (litellm/types/router.py), which the proxy's config loader also reads. +pub const SECRET_POINTER_PARAMS: &[&str] = &[ + "anthropic_issuer_signing_key_ref", + "anthropic_keycloak_client_secret_ref", +]; + +const SECRET_PREFIX: &str = "os.environ/"; + +/// How a build's deployments join the registry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum Install { + /// They become every deployment (`set_model_list`). + Replace, + /// They are added at the end (a deployment retried once its provider resolves). + Append, + /// Each replaces the deployment with its id, if any, and is added at the end + /// (`add_deployment`, an upsert). + Upsert, +} + +/// One deployment the build accepted, as the registry stores it. +#[derive(Clone, Debug, PartialEq)] +pub struct BuiltDeployment { + /// The entry index in the list the build was given. + pub index: usize, + pub spec: DeploymentSpec, +} + +/// Build `entries` (model_list entries) through `host`, in order. `resolve_secrets` resolves +/// top-level `os.environ/` references first, as `set_model_list` does; `add_deployment` and +/// upserts take their params as given. `install` takes each deployment as soon as it is +/// accepted, so it joins the registry before the next one is built, as the Python router adds +/// each in turn. +pub async fn build( + host: &dyn Host, + path: BuildPath, + entries: Vec>, + resolve_secrets: bool, + mut install: impl FnMut(&BuiltDeployment) -> Result<(), Error> + Send, +) -> Result, Error> { + let declared: Vec> = entries.iter().map(declared_id).collect(); + let mut built = Vec::new(); + for (index, mut entry) in entries.into_iter().enumerate() { + let model_name = text(entry.get("model_name")); + let mut params = match entry.remove("litellm_params") { + Some(Value::Object(params)) => params, + _ => Map::new(), + }; + if resolve_secrets { + let refs = param_refs(¶ms); + if !refs.is_empty() { + let values = host.secrets(index, &refs).await?; + for (secret, value) in refs.into_iter().zip(values) { + if let SecretAt::Param(key) = secret.at { + params.insert(key, value); + } + } + } + } + let mut info = match entry.remove("model_info") { + Some(Value::Object(info)) => info, + _ => Map::new(), + }; + let declared_id = declared[index].clone(); + if !info.contains_key("id") { + info.insert( + "id".to_owned(), + Value::String(deployment_id(&model_name, ¶ms)), + ); + } + // `set_model_list` adds one deployment per organization of a list, each with the + // organization set on the same params (so every one keeps the list's id). + let organizations = match (path, params.get("organization")) { + (BuildPath::Create, Some(Value::Array(organizations))) => organizations.clone(), + _ => vec![Value::Null], + }; + let duplicate_id = path == BuildPath::Create + && declared_id.as_ref().is_some_and(|id| { + declared + .iter() + .filter(|other| other.as_ref() == Some(id)) + .count() + > 1 + }); + for organization in organizations { + if !organization.is_null() { + params.insert("organization".to_owned(), organization); + } + let mut deployment = entry.clone(); + deployment.insert("model_name".to_owned(), Value::String(model_name.clone())); + deployment.insert("litellm_params".to_owned(), Value::Object(params.clone())); + deployment.insert("model_info".to_owned(), Value::Object(info.clone())); + if let Some(spec) = build_one( + host, + path, + index, + &deployment, + declared_id.as_deref(), + duplicate_id, + ) + .await? + { + let deployment = BuiltDeployment { index, spec }; + install(&deployment)?; + built.push(deployment); + } + } + } + Ok(built) +} + +/// One deployment's steps: validate, provider, the data-source references, added. +async fn build_one( + host: &dyn Host, + path: BuildPath, + index: usize, + entry: &Map, + declared_id: Option<&str>, + duplicate_id: bool, +) -> Result, Error> { + let ask = |entry| BuildAsk { + path, + index, + entry, + declared_id, + duplicate_id, + }; + let validated = match host.validate(&ask(entry)).await? { + Built::Entry(validated) => validated, + Built::Skip | Built::Invalid => return Ok(None), + }; + let provider = match host.provider(&ask(&validated)).await? { + Provided::Provider(provider) => provider, + Provided::Invalid => return Ok(None), + }; + let refs = data_source_refs(&validated); + if !refs.is_empty() { + let _ = host.secrets(index, &refs).await?; + } + let stored = match host.added(&ask(&validated), provider.as_deref()).await? { + Built::Entry(stored) => stored, + Built::Skip | Built::Invalid => return Ok(None), + }; + let model_name = text(stored.get("model_name")); + let mut spec: DeploymentSpec = serde_json::from_value(Value::Object(stored)) + .map_err(|source| ConfigError::InvalidDeployment { model_name, source })?; + spec.provider = provider; + Ok(Some(spec)) +} + +/// The top-level `litellm_params` values `set_model_list` resolves. +fn param_refs(params: &Map) -> Vec { + params + .iter() + .filter(|(key, _)| !SECRET_POINTER_PARAMS.contains(&key.as_str())) + .filter_map(|(key, value)| { + let name = value + .as_str() + .filter(|name| name.starts_with(SECRET_PREFIX))?; + Some(SecretRef { + name: name.to_owned(), + at: SecretAt::Param(key.clone()), + }) + }) + .collect() +} + +/// The `dataSources[].parameters.endpoint` / `key` references `_add_deployment` reads from the +/// environment. +fn data_source_refs(entry: &Map) -> Vec { + let sources = entry + .get("litellm_params") + .and_then(|params| params.get("dataSources")) + .and_then(Value::as_array); + let mut refs = Vec::new(); + for (source, data_source) in sources.into_iter().flatten().enumerate() { + let Some(parameters) = data_source.get("parameters").and_then(Value::as_object) else { + continue; + }; + for key in ["endpoint", "key"] { + if let Some(name) = parameters + .get(key) + .and_then(Value::as_str) + .filter(|name| name.starts_with(SECRET_PREFIX)) + { + refs.push(SecretRef { + name: name.to_owned(), + at: SecretAt::DataSource { + source, + key: key.to_owned(), + }, + }); + } + } + } + refs +} + +/// The `model_info.id` an entry declares, as Python's `str()` spells it. +fn declared_id(entry: &Map) -> Option { + match entry.get("model_info")?.get("id")? { + Value::Null => None, + Value::String(id) => Some(id.clone()), + other => Some(other.to_string()), + } +} + +/// A name as Python's `str()` spells it. +fn text(value: Option<&Value>) -> String { + match value { + Some(Value::String(text)) => text.clone(), + Some(Value::Null) | None => "None".to_owned(), + Some(other) => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use serde_json::json; + + use super::*; + use crate::{ + cache::{InMemoryRouterCache, RouterCache}, + error::AttemptError, + host::{Attempt, Output, RouterEvent, RouterRequest, RoutingContext, SelectStage}, + registry::Deployment, + }; + + /// Records every build step and resolves `os.environ/X` to `secret-X`. + #[derive(Default)] + struct Recording { + steps: Mutex>, + } + + #[async_trait] + impl Host for Recording { + async fn invoke(&self, _attempt: &Attempt) -> Result { + unreachable!("the build runs no attempt") + } + async fn select( + &self, + _stage: &SelectStage, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + Ok(candidates) + } + async fn allow_fallback( + &self, + _request: &RouterRequest, + _target: &str, + ) -> Result { + Ok(true) + } + async fn sleep(&self, _duration: std::time::Duration) {} + fn observe(&self, _event: &RouterEvent) {} + fn cache(&self) -> Arc { + Arc::new(InMemoryRouterCache::default()) + } + async fn secrets(&self, _index: usize, refs: &[SecretRef]) -> Result, Error> { + self.steps.lock().unwrap().push(format!("secrets {refs:?}")); + Ok(refs + .iter() + .map(|r| json!(format!("secret-{}", &r.name[SECRET_PREFIX.len()..]))) + .collect()) + } + async fn validate(&self, ask: &BuildAsk<'_>) -> Result { + self.steps.lock().unwrap().push(format!( + "validate {} {}", + ask.index, ask.entry["litellm_params"] + )); + Ok(Built::Entry(ask.entry.clone())) + } + async fn provider(&self, ask: &BuildAsk<'_>) -> Result { + self.steps + .lock() + .unwrap() + .push(format!("provider {}", ask.index)); + Ok(Provided::Provider(Some("azure".to_owned()))) + } + async fn added(&self, ask: &BuildAsk<'_>, provider: Option<&str>) -> Result { + self.steps + .lock() + .unwrap() + .push(format!("added {} {provider:?}", ask.index)); + Ok(Built::Entry(ask.entry.clone())) + } + } + + fn entries(value: Value) -> Vec> { + serde_json::from_value(value).unwrap() + } + + #[tokio::test] + async fn create_resolves_secrets_then_builds_one_deployment_per_organization() { + let host = Recording::default(); + let built = build( + &host, + BuildPath::Create, + entries(json!([{ + "model_name": "g", + "litellm_params": { + "model": "azure/x", + "api_key": "os.environ/KEY", + "anthropic_issuer_signing_key_ref": "os.environ/POINTER", + "organization": ["a", "b"], + }, + }])), + true, + |_| Ok(()), + ) + .await + .unwrap(); + let steps = host.steps.lock().unwrap().clone(); + assert_eq!(steps.len(), 7, "{steps:?}"); + assert!(steps[0].starts_with("secrets") && steps[0].contains("KEY")); + assert!(!steps[0].contains("POINTER")); + assert!(steps[1].contains(r#""organization":"a""#)); + assert!(steps[1].contains("secret-KEY")); + assert_eq!(steps[3], "added 0 Some(\"azure\")"); + assert!(steps[4].contains(r#""organization":"b""#)); + assert_eq!(built.len(), 2); + assert_eq!(built[0].spec.provider.as_deref(), Some("azure")); + assert_eq!( + built[0].spec.model_info["id"], + built[1].spec.model_info["id"] + ); + let deployment = Deployment::from_spec(built[0].spec.clone()).unwrap(); + assert_eq!(deployment.provider(), "azure"); + } + + #[tokio::test] + async fn add_leaves_params_as_given_and_asks_for_data_source_references() { + let host = Recording::default(); + let built = build( + &host, + BuildPath::Add, + entries(json!([{ + "model_name": "g", + "litellm_params": { + "model": "azure/x", + "api_key": "os.environ/KEY", + "organization": ["a", "b"], + "dataSources": [{"parameters": {"endpoint": "os.environ/END", "key": "plain"}}], + }, + "model_info": {"id": "given"}, + }])), + false, + |_| Ok(()), + ) + .await + .unwrap(); + let steps = host.steps.lock().unwrap().clone(); + assert_eq!(steps.len(), 4, "{steps:?}"); + assert!(steps[0].contains("os.environ/KEY")); + assert!(steps[2].contains("DataSource { source: 0, key: \"endpoint\" }")); + assert_eq!(built.len(), 1); + assert_eq!(built[0].spec.model_info["id"], "given"); + } +} diff --git a/litellm-rust/crates/router-core/src/cache.rs b/litellm-rust/crates/router-core/src/cache.rs new file mode 100644 index 00000000000..9811e34a3e3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/cache.rs @@ -0,0 +1,159 @@ +//! [`RouterCache`]: the DualCache-style store all routing state lives in, under the keys, +//! encodings and TTLs the Python router uses, so Python and Rust workers share it. Method names +//! follow `litellm-cache`'s `BaseCache`, `BatchCache`, `CounterCache` and `DeleteCache`. + +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use litellm_cache::Error; +use parking_lot::Mutex; +use serde_json::Value; + +#[async_trait] +pub trait RouterCache: Send + Sync { + async fn async_get_cache(&self, key: &str) -> Result, Error>; + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), Error>; + + /// Add `amount` to the number at `key` (0 when missing) and return the new value. The TTL + /// applies when the key is created, as Redis `INCRBYFLOAT` + `EXPIRE NX` does. + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result; + + async fn async_batch_get_cache(&self, keys: &[String]) -> Result>, Error>; + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error>; + + /// Read this worker's in-memory tier only (`local_only=True`). + async fn async_get_local(&self, key: &str) -> Result, Error> { + self.async_get_cache(key).await + } + + /// Add to a counter in this worker's in-memory tier only (`local_only=True`). + async fn async_increment_local( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + self.async_increment(key, amount, ttl).await + } + + /// `_increment_deployment_usage`'s write: add each `(key, amount)` in one pipeline. With a + /// shared tier, this worker's in-memory tier then holds the shared totals, so its reads see + /// what other workers counted up to now. + async fn async_increment_usage( + &self, + increments: &[(String, f64)], + ttl: Option, + ) -> Result<(), Error> { + for (key, amount) in increments { + self.async_increment(key, *amount, ttl).await?; + } + Ok(()) + } +} + +#[derive(Default)] +struct Entry { + value: Value, + expires: Option, +} + +/// An in-process [`RouterCache`], for tests and for state Python keeps `local_only`. +#[derive(Clone, Default)] +pub struct InMemoryRouterCache { + entries: Arc>>, +} + +impl InMemoryRouterCache { + fn live(entries: &mut HashMap, key: &str) -> Option { + let expired = entries + .get(key) + .is_some_and(|entry| entry.expires.is_some_and(|at| at <= Instant::now())); + if expired { + entries.remove(key); + } + entries.get(key).map(|entry| entry.value.clone()) + } + + pub fn get(&self, key: &str) -> Option { + Self::live(&mut self.entries.lock(), key) + } + + pub fn increment(&self, key: &str, amount: f64, ttl: Option) -> f64 { + let mut entries = self.entries.lock(); + let current = Self::live(&mut entries, key).and_then(|value| value.as_f64()); + let next = current.unwrap_or(0.0) + amount; + let value = serde_json::Number::from_f64(next) + .map(|number| { + if next.fract() == 0.0 && next.abs() < i64::MAX as f64 { + Value::from(next as i64) + } else { + Value::Number(number) + } + }) + .unwrap_or(Value::Null); + let expires = match (current, entries.get(key)) { + (Some(_), Some(entry)) => entry.expires, + _ => ttl.map(|ttl| Instant::now() + ttl), + }; + entries.insert(key.to_owned(), Entry { value, expires }); + next + } +} + +#[async_trait] +impl RouterCache for InMemoryRouterCache { + async fn async_get_cache(&self, key: &str) -> Result, Error> { + Ok(self.get(key)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), Error> { + let expires = ttl.map(|ttl| Instant::now() + ttl); + self.entries + .lock() + .insert(key.to_owned(), Entry { value, expires }); + Ok(()) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + Ok(self.increment(key, amount, ttl)) + } + + async fn async_batch_get_cache(&self, keys: &[String]) -> Result>, Error> { + let mut entries = self.entries.lock(); + Ok(keys + .iter() + .map(|key| Self::live(&mut entries, key)) + .collect()) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { + self.entries.lock().remove(key); + Ok(()) + } +} diff --git a/litellm-rust/crates/router-core/src/call.rs b/litellm-rust/crates/router-core/src/call.rs new file mode 100644 index 00000000000..287bb98d8b7 --- /dev/null +++ b/litellm-rust/crates/router-core/src/call.rs @@ -0,0 +1,1555 @@ +//! The call loop: select, run the attempt, then retry in the same group or fall back to another +//! until it succeeds or the policy is exhausted. Mirrors `Router.async_function_with_fallbacks`, +//! `async_function_with_fallbacks_common_utils`, `async_function_with_retries`, +//! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and +//! `run_async_fallback` in `router_utils/fallback_event_handlers.py`. + +use std::{ + borrow::Cow, + future::Future, + pin::Pin, + sync::Arc, + time::{Duration, Instant}, +}; + +use serde_json::{Map, Value}; + +use tracing::warn; + +use crate::{ + Core, RouterStateSnapshot, + config::Op, + error::{AttemptError, ConfigError, Error, FailureKind, Unsupported}, + fallback::{FallbackKeys, FallbackTarget, Fallbacks}, + health::{GroupShape, should_retry_status, team_model_has_alternatives}, + host::{ + Attempt, Blocked, Output, PickError, RetryCause, RouterEvent, RouterRequest, + UncountedRetries, + }, + host_index::python_str, + lookup::Lookup, + mock::MockFailure, + outcome::OutcomeWrite, + prerouting::{PreRoute, PreRouted}, + registry::{ResolveKind, ResolveScope}, + retry::{RetryContext, resolve_retry_policy, should_retry}, + select::{Candidates, SelectInputs}, +}; + +mod stream; + +pub use self::stream::{StreamOrigin, StreamPolicy}; + +/// `INITIAL_RETRY_DELAY`, `MAX_RETRY_DELAY` and `JITTER` in `litellm/constants.py`. +const INITIAL_RETRY_DELAY: f64 = 0.5; +const MAX_RETRY_DELAY: f64 = 8.0; +const JITTER: f64 = 0.75; +/// `calculate_retry_after` obeys a provider's retry-after up to a minute. +const MAX_HONOURED_RETRY_AFTER: Duration = Duration::from_secs(60); + +#[derive(Clone, Debug)] +pub struct RouterResponse { + pub output: Output, + pub deployment_id: String, + pub model_group: String, + pub attempted_retries: u32, + /// The retry budget of the loop a retry succeeded in (`x-litellm-max-retries`); `None` + /// when the first attempt succeeded. + pub max_retries: Option, + pub attempted_fallbacks: u32, +} + +/// A failed attempt with the deployment it ran on, when one was picked. +struct Failed { + error: Box, + deployment_id: Option, +} + +impl From for Failed { + fn from(error: Error) -> Self { + Self { + error: Box::new(error), + deployment_id: None, + } + } +} + +/// Where a level of the fallback chain sits: `depth` is `run_async_fallback`'s counter, one more +/// for every target tried, `level` how many fallbacks deep the chain is, and `scope` the level's +/// own copy of the request metadata (every hop copies its parent's). +#[derive(Clone, Copy, Debug, Default)] +struct FallbackHop { + depth: u32, + level: u32, + scope: u32, +} + +/// What the whole chain shares: the attempts made (`attempted_targets`) and the scopes handed out. +#[derive(Debug, Default)] +struct FallbackChain { + attempted: Vec, + scopes: u32, +} + +/// What one level of the chain serves: a group, or the same group at a later `order`, with the +/// request params the chain's dict targets merged so far (`kwargs.update(target)`). +#[derive(Clone, Debug)] +struct Target { + group: String, + order: Option, + params: Option>>, + /// `_excluded_deployment_ids` a weighted failover hop put on the request: never picked. + excluded: Arc<[String]>, + /// `_failover_excluded_ids` in the level's metadata: the deployments weighted failover + /// already excluded on the way here. + failover_excluded: Arc<[String]>, +} + +/// A step of a level's fallback chain: a configured target, or an order level of the group. +#[derive(Clone, Debug)] +enum Next { + Fallback(FallbackTarget), + Order(String, i64), + /// A weighted failover hop: the same group without the deployments already excluded. + Failover(String, Arc<[String]>), +} + +/// An attempt the chain already made (`fallback_attempt_key`). +#[derive(Clone, Debug, PartialEq)] +enum Tried { + Target(FallbackTarget), + Order(String, i64), + Failover(String, Vec), +} + +/// The fallback lists a call reads: the request's own where it passed them, else the router's. +struct Lists<'a> { + fallbacks: Option<&'a Fallbacks>, + context_window: Option<&'a Fallbacks>, + content_policy: Option<&'a Fallbacks>, + keys: &'a FallbackKeys, +} + +impl<'a> Lists<'a> { + fn of(snapshot: &'a RouterStateSnapshot, request: &'a RouterRequest) -> Self { + let dynamic = &snapshot.settings.dynamic; + let pick = |own: &'a Option>, router: Option<&'a Fallbacks>| match own { + Some(own) => own.as_ref(), + None => router, + }; + Self { + fallbacks: pick(&request.fallbacks, Some(&dynamic.fallbacks)), + context_window: pick( + &request.context_window_fallbacks, + dynamic.context_window_fallbacks.as_ref(), + ), + content_policy: pick( + &request.content_policy_fallbacks, + dynamic.content_policy_fallbacks.as_ref(), + ), + keys: &dynamic.fallback_keys, + } + } + + fn has_regular(&self) -> bool { + self.fallbacks.is_some_and(|list| !list.is_empty()) + } +} + +/// `has_unattempted_fallback_target` over the chain `list` resolves for the level's lookup +/// groups: before the first hop nothing counts as tried, and a dict target with its own params +/// is never skipped. +fn unattempted_chain( + list: &Fallbacks, + lists: &Lists<'_>, + request: &RouterRequest, + target: &Target, + at: FallbackHop, + attempted: &[Tried], +) -> bool { + let mut lookup = vec![target.group.as_str()]; + if request.model != target.group { + lookup.push(request.model.as_str()); + } + list.lookup_groups(&lookup, lists.keys) + .is_some_and(|targets| { + at.level == 0 + || targets.iter().any(|next| match next.identity() { + FallbackTarget::Params(_) => true, + identity => !attempted.contains(&Tried::Target(identity)), + }) + }) +} + +/// The deployments `_core_get_model_list(model_name=group, team_id=team_id)` lists: the name's own +/// (a team's public name among them), an alias's target's, a routing group's members, and only +/// when none of those serve it, the wildcard routes (`_get_wildcard_deployments`), the proxy-wide +/// pattern's and then the team's. +fn group_members( + snapshot: &RouterStateSnapshot, + group: &str, + team_id: Option<&str>, +) -> Vec> { + let registry = &snapshot.registry; + let all = |name: &str, team_id: Option<&str>| { + crate::resolve::all_deployments(registry, name, team_id).unwrap_or_default() + }; + let members = |routing_group: &crate::registry::RoutingGroupSpec| { + crate::routing_groups::members(routing_group, |member| all(member, None)) + }; + let mut rows = all(group, team_id); + if let Some(alias_target) = registry.alias_of(group) { + rows.extend(match registry.routing_group_named(alias_target) { + Some(routing_group) => members(routing_group), + None => all(alias_target, None), + }); + } + if let Some(routing_group) = registry.routing_group_named(group) { + rows.extend(members(routing_group)); + } + if rows.is_empty() { + if let Some((_, routed)) = registry.wildcard_for(group) { + rows.extend(routed); + } + if let Some(team) = team_id { + rows.extend(Lookup::new(registry).team_wildcard(group, team)); + } + } + rows +} + +/// What `_maybe_run_weighted_failover` catches and defers to the regular fallbacks: +/// `openai.APIError` (every litellm provider error, a failure the host reported no classes for +/// among them), `RouterRateLimitError` and `RouterRateLimitErrorBasic`. Anything else propagates. +fn defers_to_regular_fallbacks(error: &Error) -> bool { + retried_failure(error).is_some_and(|failure| { + failure.classes.is_empty() + || [ + "APIError", + "BadRequestError", + "ServiceUnavailableError", + "RouterRateLimitError", + "RouterRateLimitErrorBasic", + ] + .iter() + .any(|class| failure.is_a(class)) + }) +} + +/// `is_bad_request_without_fallback` with `fallback_on_unsupported_params` off: an unsupported +/// params error, or a provider's plain 400 bad request. +fn is_bad_request_without_fallback(failure: &AttemptError) -> bool { + if !failure.is_a("BadRequestError") { + return false; + } + if failure.is_a("UnsupportedParamsError") { + return true; + } + if [ + "ContextWindowExceededError", + "ContentPolicyViolationError", + "UnsupportedEndpointError", + ] + .iter() + .any(|class| failure.is_a(class)) + { + return false; + } + failure.status == Some(400) && failure.names_provider +} + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// `KeyError`'s classes, most specific first. +/// The failure a retry loop decides by: an attempt's, or the exception a failed pick raised +/// (Python's loop retries a pick's error like any other: no deployment available, a stale +/// routing group's `KeyError`, ...). +fn retried_failure(error: &Error) -> Option> { + match error { + Error::Attempt(failure) => Some(Cow::Borrowed(failure)), + _ => error.pick_failure().map(Cow::Owned), + } +} + +impl Core { + pub(crate) async fn call( + &self, + op: Op, + request: Arc, + ) -> Result { + let snapshot = self.snapshot.load_full(); + let mut chain = FallbackChain::default(); + let target = Target { + group: request.model.clone(), + order: None, + params: None, + excluded: Arc::from([]), + failover_excluded: Arc::from([]), + }; + let at = FallbackHop::default(); + // A fallbacks mock stands in for the requested group's retry loop: no pick, no attempt. + let Some(mock) = request.mock_failure.filter(|mock| mock.before_retries()) else { + return self + .with_fallbacks(&snapshot, &op, &request, target, at, &mut chain) + .await; + }; + let failed = Failed::from(self.mock_failed(mock, &target, at, None)); + self.fall_back(&snapshot, &op, &request, target, at, &mut chain, failed) + .await + } + + /// One level of `async_function_with_fallbacks`: the target's retry loop, then its + /// fallbacks if every attempt failed. + #[allow(clippy::too_many_arguments)] + fn with_fallbacks<'a>( + &'a self, + snapshot: &'a RouterStateSnapshot, + op: &'a Op, + request: &'a Arc, + target: Target, + at: FallbackHop, + chain: &'a mut FallbackChain, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let failed = match self + .with_retries(snapshot, op, request, &target, at, &chain.attempted) + .await + { + Ok(response) => return Ok(response), + Err(failed) => failed, + }; + self.fall_back(snapshot, op, request, target, at, chain, failed) + .await + }) + } + + /// The level after its own failure: the no-fallback checks, weighted failover, then the + /// fallback targets. A level that fails raises its own error, whatever its fallbacks raised, + /// except an order chain's failure on any error but a context window one, which raises what + /// the chain's last hop raised ([`RouterEvent::OrderChainFailed`]). + /// A stream that fails before content re-enters here (`fallback_stream`), so the group's + /// retry loop does not run again. + #[allow(clippy::too_many_arguments)] + async fn fall_back( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: Target, + at: FallbackHop, + chain: &mut FallbackChain, + failed: Failed, + ) -> Result { + let Failed { + error, + deployment_id: failed_id, + } = failed; + let error = *error; + self.level_failed(at, &target, &error, false); + if self.raises_without_fallback(snapshot, request, &error) { + return Err(error); + } + // `async_function_with_fallbacks_common_utils` lists the group's deployments + // (`_core_get_model_list`) for its order fallbacks: a written index position past the + // model list raises there, before any fallback. + if let Err(raised) = crate::resolve::all_deployments( + &snapshot.registry, + &target.group, + request.listing_team_id.as_deref(), + ) { + self.level_failed(at, &target, &raised, true); + return Err(raised); + } + let (targets, order_path) = self.fallback_targets(snapshot, request, &target, &error); + if !order_path { + self.host.observe(&RouterEvent::FallingBack { + level: at.level, + model: target.group.clone(), + }); + } + let mut target = target; + // `_maybe_run_weighted_failover`: its own `run_async_fallback`, before the regular one. + let failover = if order_path { + None + } else { + match self + .weighted_failover(snapshot, request, &target, &error, failed_id.as_deref()) + .await + { + Ok(failover) => failover, + Err(raised) => { + // `_get_routing_context` raised inside the level's fallback handling: the + // level raises that error, before any regular fallback. + self.level_failed(at, &target, &raised, true); + return Err(raised); + } + } + }; + if let Some(failover) = failover { + let excluded = match &failover { + Next::Failover(_, excluded) => excluded.clone(), + _ => Arc::from([]), + }; + target.failover_excluded = excluded; + match self + .run_fallbacks(snapshot, op, request, &target, at, chain, vec![failover]) + .await + { + Ok(response) => return Ok(response), + Err(raised) if defers_to_regular_fallbacks(raised.as_ref().unwrap_or(&error)) => {} + // Outside the `try` around the regular fallbacks: the level raises it as it is. + Err(raised) => { + self.host.observe(&RouterEvent::OrderChainFailed { + level: at.level, + last_hop: raised.is_some(), + }); + return Err(raised.unwrap_or(error)); + } + } + } + let Some(targets) = targets else { + self.host.observe(&RouterEvent::FallbacksFailed { + level: at.level, + chain: false, + last_hop: false, + }); + return Err(error); + }; + let last_hop_error = match self + .run_fallbacks(snapshot, op, request, &target, at, chain, targets) + .await + { + Ok(response) => return Ok(response), + Err(raised) => raised, + }; + if !order_path { + self.host.observe(&RouterEvent::FallbacksFailed { + level: at.level, + chain: true, + last_hop: last_hop_error.is_some(), + }); + return Err(error); + } + // The order path's `raise`, outside the `try` that re-raises the original error: only a + // context window error stays the one the caller sees. + let context_window = retried_failure(&error) + .is_some_and(|failure| failure.is_a("ContextWindowExceededError")); + let raised = last_hop_error.filter(|_| !context_window); + self.host.observe(&RouterEvent::OrderChainFailed { + level: at.level, + last_hop: raised.is_some(), + }); + Err(raised.unwrap_or(error)) + } + + /// `run_async_fallback` over one chain: the depth is checked once on entry, then every + /// target tried counts one more. When the chain served nothing, the error its last hop + /// raised (`error_from_fallbacks`), or `None` when no hop ran. + #[allow(clippy::too_many_arguments)] + fn run_fallbacks<'a>( + &'a self, + snapshot: &'a RouterStateSnapshot, + op: &'a Op, + request: &'a Arc, + target: &'a Target, + at: FallbackHop, + chain: &'a mut FallbackChain, + targets: Vec, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let max_fallbacks = request + .max_fallbacks + .unwrap_or(snapshot.settings.max_fallbacks); + if at.depth >= max_fallbacks { + return Err(None); + } + let failed = Tried::Target(FallbackTarget::Name(target.group.clone())); + if !chain.attempted.contains(&failed) { + chain.attempted.push(failed); + } + // Before the loop, or the refusal would be one target's failure: no target of the + // chain is tried, and the walk raises a `ValueError` the host holds. + let dicts: Vec> = targets + .iter() + .filter_map(|next| match next { + Next::Fallback(FallbackTarget::Params(dict)) => Some(dict.clone()), + _ => None, + }) + .collect(); + if !dicts.is_empty() && self.host.refuses_fallbacks(at.level, &dicts).await { + let refused = AttemptError::host_raised(&["ValueError"]); + return Err(Some(Error::Attempt(refused))); + } + let mut params = target.params.clone(); + let mut hop = at.depth; + let mut last_hop_error = None; + // `kwargs["model"]` as a dict target's non-string `model` left it: later targets + // without a `model` keep it. + let mut unroutable: Option = None; + for next in targets { + let (tried, group, order) = match &next { + Next::Order(group, order) => ( + Tried::Order(group.clone(), *order), + group.clone(), + Some(*order), + ), + Next::Failover(group, excluded) => { + let mut key = excluded.to_vec(); + key.sort(); + (Tried::Failover(group.clone(), key), group.clone(), None) + } + Next::Fallback(FallbackTarget::Name(name)) if *name == target.group => continue, + // A provider-scoped request never leaves its group; a dict target with no + // `model` names no group, so it is skipped too. + Next::Fallback(fallback) + if request.same_group_fallbacks_only + && fallback.model() != Some(target.group.as_str()) => + { + continue; + } + Next::Fallback(fallback) => { + if let Some(model) = fallback.model() + && model != target.group + { + // A check that raises ends the loop outside its `try`, as the + // chain's own failure. + match self.host.allow_fallback(request, model).await { + Ok(true) => {} + Ok(false) => continue, + Err(raised) => return Err(Some(Error::Attempt(raised))), + } + } + let group = fallback.model().unwrap_or(&target.group).to_owned(); + (Tried::Target(fallback.identity()), group, None) + } + }; + if chain.attempted.contains(&tried) { + continue; + } + chain.attempted.push(tried); + if let Next::Fallback(fallback) = &next + && let Some(merged) = fallback.params() + { + let mut combined = params.as_deref().cloned().unwrap_or_default(); + combined.extend(merged.iter().map(|(k, v)| (k.clone(), v.clone()))); + params = Some(Arc::new(combined)); + } + match &next { + Next::Fallback(fallback) if fallback.model().is_none() => { + if let Some(model) = fallback.unroutable_model() { + unroutable = Some(model.clone()); + } + } + _ => unroutable = None, + } + let group = unroutable.as_ref().map_or(group, python_str); + hop += 1; + self.host.observe(&RouterEvent::RetryLogged { + scope: at.scope, + level: at.level, + model: target.group.clone(), + attempted_retries: None, + cause: RetryCause::Level, + }); + chain.scopes += 1; + self.host.observe(&RouterEvent::Fallback { + from: target.group.clone(), + to: group.clone(), + order, + depth: hop, + level: at.level + 1, + scope: chain.scopes, + parent: at.scope, + max_fallbacks, + unroutable: unroutable.clone(), + }); + let excluded = match &next { + Next::Failover(_, excluded) => excluded.clone(), + _ => target.excluded.clone(), + }; + let next = Target { + group: group.clone(), + order, + params: params.clone(), + excluded, + failover_excluded: target.failover_excluded.clone(), + }; + let parent = at; + let at = FallbackHop { + depth: hop, + level: at.level + 1, + scope: chain.scopes, + }; + // A non-string `model` is a hop that fails before any attempt, on an error its + // own level's fallback lookup raises too, so it walks no chain of its own. + let result = match &unroutable { + Some(_) => Err(Error::UnknownModel { + model: group.clone(), + }), + None => { + self.with_fallbacks(snapshot, op, request, next, at, chain) + .await + } + }; + self.host.observe(&RouterEvent::FallbackDone { + from: target.group.clone(), + to: group, + level: parent.level, + scope: at.scope, + parent: parent.scope, + succeeded: result.is_ok(), + }); + match result { + Ok(mut response) => { + response.attempted_fallbacks = hop; + return Ok(response); + } + Err(raised) => last_hop_error = Some(raised), + } + } + Err(last_hop_error) + }) + } + + /// `_maybe_run_weighted_failover`'s hop for a failed deployment, when it applies: the + /// simple-shuffle group re-picked without every deployment already excluded, while one + /// that is not cooling down remains. + async fn weighted_failover( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + error: &Error, + failed_id: Option<&str>, + ) -> Result, Error> { + if !snapshot.settings.enable_weighted_failover { + return Ok(None); + } + if retried_failure(error).is_some_and(|failure| { + failure.is_a("ContextWindowExceededError") + || failure.is_a("ContentPolicyViolationError") + }) { + return Ok(None); + } + // `_get_routing_context` runs first, whatever failed: a stale routing-group member map + // raises its `KeyError` here. + snapshot.missing_routing_group(request, &target.group)?; + if request.skip_weighted_failover || error.attempt().is_none() { + return Ok(None); + } + let kind = snapshot + .registry + .resolve(&target.group, ResolveScope::default()) + .map_or(ResolveKind::Group, |resolution| resolution.kind); + let (strategy, _) = crate::strategy::routing_context( + &snapshot.registry, + &snapshot.settings.routing_strategy, + request.routing_strategy.as_deref(), + &kind, + &target.group, + ); + if strategy != crate::config::RoutingStrategy::SimpleShuffle { + return Ok(None); + } + let Some(failed_id) = failed_id else { + return Ok(None); + }; + let mut excluded: Vec = target.failover_excluded.to_vec(); + if !excluded.iter().any(|id| id == failed_id) { + excluded.push(failed_id.to_owned()); + } + let listing_team = request.listing_team_id.as_deref(); + let ids: Vec = group_members(snapshot, &target.group, listing_team) + .iter() + .filter(|d| d.listed_for_team(listing_team)) + .map(|d| d.id.clone()) + .collect(); + let cooling = self + .health + .active_cooldowns(&snapshot.registry.ids()) + .await + .unwrap_or_default(); + let remaining = ids + .iter() + .any(|id| !excluded.contains(id) && !cooling.iter().any(|(cooled, _)| cooled == id)); + Ok(remaining.then(|| Next::Failover(target.group.clone(), excluded.into()))) + } + + /// `async_function_with_fallbacks_common_utils` raises before looking for any fallback: + /// fallbacks are off, the error is a guardrail's, a bad request without + /// `fallback_on_unsupported_params`, or no fallback serves the error at all. + fn raises_without_fallback( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + error: &Error, + ) -> bool { + if request.disable_fallbacks { + return true; + } + // Python decides by the exception alone, so a pick's error falls back like an attempt's: + // an `UnsupportedParamsError` raises, and the refusals name no provider. Only an error no + // pick raises (no Python exception stands behind it) has no fallback. + retried_failure(error).is_none_or(|failure| { + failure.kind == FailureKind::GuardrailIntervention + || (!snapshot.settings.fallback_on_unsupported_params + && is_bad_request_without_fallback(&failure)) + }) + } + + /// The fallback targets for a failure, or `None` when it must be raised as is + /// (`async_function_with_fallbacks_common_utils`). + fn fallback_targets( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + error: &Error, + ) -> (Option>, bool) { + // A pick's error decides by its exception's classes as an attempt's does: a pre-call + // context window refusal takes the context window fallbacks, any other the generic ones. + let failure = retried_failure(error); + let is_a = |name: &str| failure.as_ref().is_some_and(|f| f.is_a(name)); + let lists = Lists::of(snapshot, request); + // `fallback_lookup_groups`: the group being served, then the group the request was for. + let mut lookup = vec![target.group.as_str()]; + if request.model != target.group { + lookup.push(request.model.as_str()); + } + let chain = |list: &Fallbacks| { + if list.is_client_side(lists.keys) { + Some(list.as_chain()) + } else { + list.lookup_groups(&lookup, lists.keys) + } + }; + let fallbacks_of = |targets: Vec| { + targets.into_iter().map(Next::Fallback).collect::>() + }; + // `context_window_fallback_model_group`: only a chain the group matched takes precedence. + let context_window_targets = is_a("ContextWindowExceededError") + .then(|| lists.context_window?.lookup_groups(&lookup, lists.keys)) + .flatten(); + let skip_order = is_a("ContentPolicyViolationError") || context_window_targets.is_some(); + if !skip_order { + // A routing group's members, with a priority group's priorities as their order. + let listing_team = request.listing_team_id.as_deref(); + let mut orders: Vec = group_members(snapshot, &target.group, listing_team) + .iter() + .filter(|d| d.listed_for_team(listing_team)) + .filter_map(|d| d.order()) + .collect(); + orders.sort_unstable(); + orders.dedup(); + if orders.len() > 1 { + let current = target.order.unwrap_or(orders[0]); + let mut targets: Vec = orders + .into_iter() + .filter(|order| *order > current) + .map(|order| Next::Order(target.group.clone(), order)) + .collect(); + if let Some(list) = lists.fallbacks { + targets.extend(fallbacks_of(chain(list).unwrap_or_default())); + } + if !targets.is_empty() { + return (Some(targets), true); + } + } + } + if let Some(list) = lists.fallbacks + && list.is_client_side(lists.keys) + && context_window_targets.is_none() + { + return (Some(fallbacks_of(list.as_chain())), false); + } + if let Some(targets) = context_window_targets { + return (Some(fallbacks_of(targets)), false); + } + // A context window list with no match falls through to the regular fallbacks; a content + // policy list with no match raises. + if is_a("ContentPolicyViolationError") + && let Some(list) = lists.content_policy + { + return ( + list.lookup_groups(&lookup, lists.keys).map(fallbacks_of), + false, + ); + } + ( + lists + .fallbacks + .and_then(|list| list.lookup_groups(&lookup, lists.keys)) + .map(fallbacks_of), + false, + ) + } + + /// A level's first attempt, or the rate-limit mock in its place + /// (`_handle_mock_testing_rate_limit_error`), which the retry loop handles like a failed + /// attempt. + #[allow(clippy::too_many_arguments)] + async fn first_attempt( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + inputs: &SelectInputs, + max_retries: u32, + place: (FallbackHop, &[Tried]), + ) -> Result { + if request.mock_failure != Some(MockFailure::RateLimit) { + return self + .attempt( + snapshot, + op, + request, + target, + inputs, + (0, max_retries), + place, + ) + .await; + } + // A lone deployment's `num_retries` rides on the mock error, as on a real one. + let lone = snapshot + .registry + .group(&target.group) + .filter(|group| group.len() == 1) + .and_then(|group| group[0].raw_params.get("num_retries")?.as_u64()) + .and_then(|retries| u32::try_from(retries).ok()); + Err(self + .mock_failed(MockFailure::RateLimit, target, place.0, lone) + .into()) + } + + async fn with_retries( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + at: FallbackHop, + attempted: &[Tried], + ) -> Result { + let place = (at, attempted); + let mut inputs = SelectInputs { + target_order: target.order, + excluded_ids: target.excluded.clone(), + requested: Some(target.group.clone()), + fallback_hop: at.level > 0, + ..SelectInputs::default() + }; + let counting = SelectInputs { + count_only: true, + ..SelectInputs::default() + }; + let settings = &snapshot.settings; + let first_budget = request.num_retries.unwrap_or(settings.num_retries); + self.host.observe(&RouterEvent::RetriesOpened { + scope: at.scope, + max_retries: first_budget, + model_group_size: snapshot.registry.model_list_len(&target.group), + }); + let first = self + .first_attempt(snapshot, op, request, target, &inputs, first_budget, place) + .await; + let mut failed = match first { + Ok(response) => return Ok(response), + Err(failed) => failed, + }; + self.pick_failed(at, target, &failed.error); + // A guardrail's block and a failed agentic follow-up are re-raised as they are, before + // anything else looks at them (`async_function_with_retries`). + if matches!( + &*failed.error, + Error::Attempt(failure) + if failure.kind == FailureKind::GuardrailIntervention || failure.agentic_followup + ) { + return Err(failed); + } + // The first failure looks the group's retry policy up; a lookup that raises replaces the + // failure with its own error, before any retry (`async_function_with_retries`). + if request.retry_policy_raises.contains(&target.group) { + self.host.observe(&RouterEvent::RetryPolicyRaised { + level: at.level, + model: target.group.clone(), + cause: RetryCause::of(&failed.error), + }); + return Err(Failed { + error: Box::new(Error::Attempt(AttemptError::host_raised(&["Exception"]))), + deployment_id: None, + }); + } + let Some(first_failure) = retried_failure(&failed.error) else { + return Err(failed); + }; + let failure: &AttemptError = &first_failure; + // A request's `num_retries`, a count or not, wins over the deployment's and the router's. + let mut num_retries = if request.uncounted_retries.is_some() { + 0 + } else { + request + .num_retries + .or(failure.num_retries) + .unwrap_or(settings.num_retries) + }; + let group_policies = request + .model_group_retry_policy + .as_ref() + .unwrap_or(&settings.model_group_retry_policy); + let policy_retries = (request.num_retries != Some(0)) + .then(|| { + resolve_retry_policy( + settings.retry_policy.as_ref(), + &target.group, + Some(group_policies), + )? + .retries_for(failure) + }) + .flatten(); + let policy_applies = policy_retries.is_some(); + num_retries = policy_retries.unwrap_or(num_retries); + drop(first_failure); + // `_async_get_healthy_deployments` finds no group for a deployment id, so a call by id + // counts no deployments when deciding whether to retry. + let counted = |c: &Candidates, n: usize| if c.kind.is_single() { 0 } else { n }; + let all = self + .candidates(snapshot, op, request, &target.group, &counting) + .await + .map(|c| counted(&c, c.all.len())) + .unwrap_or(0); + + for retry in 0..=num_retries { + if retry > 0 { + self.log_retry(target, at, retry, &failed.error); + } + let healthy_ids: Vec = match self + .candidates(snapshot, op, request, &target.group, &counting) + .await + { + Ok(c) => { + let ids = c.healthy.iter().map(|d| d.id.clone()); + ids.take(counted(&c, c.healthy.len())).collect() + } + // `_async_get_healthy_deployments` swallows the lookup's error and still reads the + // cooldowns: a Redis round trip the refused pick's failure log runs across. + Err(_) => { + let _ = self.health.active_cooldowns(&snapshot.registry.ids()).await; + Vec::new() + } + }; + let healthy = healthy_ids.len(); + // A pick's error is retried as Python's loop retries any exception: a retry policy + // keeps going, else `should_retry_this_error` decides (it raises when nothing is + // healthy). + let Some(failure) = retried_failure(&failed.error) else { + return Err(failed); + }; + if !policy_applies && !self.should_retry(snapshot, request, &failure, healthy, all) { + return Err(failed); + } + let uncounted = request.uncounted_retries.filter(|_| !policy_applies); + if retry == 0 && uncounted.is_none() { + self.host.observe(&RouterEvent::RetryBudget { + scope: at.scope, + max_retries: num_retries, + }); + } + if uncounted == Some(UncountedRetries::Raises) { + self.host.observe(&RouterEvent::RetryCountRaised { + level: at.level, + model: target.group.clone(), + cause: RetryCause::of(&failed.error), + }); + return Err(Failed { + error: Box::new(Error::Attempt(AttemptError::host_raised(&["Exception"]))), + deployment_id: None, + }); + } + if num_retries == 0 { + return Err(failed); + } + if retry == 0 { + self.log_retry(target, at, 0, &failed.error); + } + let named = failure.failed_deployment_id.as_ref(); + if let (Some(status), Some(id)) = + (failure.status, named.or(failed.deployment_id.as_ref())) + && !should_retry_status(status) + && !inputs.skipped_ids.contains(id) + { + inputs.skipped_ids.push(id.clone()); + } + // An endpoint a deployment does not serve is retried on the others, until every + // one refused (`_every_deployment_refused`). + if failure.is_a("UnsupportedEndpointError") + && healthy_ids.iter().all(|id| inputs.skipped_ids.contains(id)) + { + return Err(failed); + } + let delay = self.time_to_sleep( + snapshot, + failure.retry_after, + num_retries - retry, + num_retries, + healthy, + all, + ); + self.host.sleep(delay).await; + if retry == num_retries { + self.host.observe(&RouterEvent::RetriesExhausted { + model_group: target.group.clone(), + attempted: num_retries, + max: num_retries, + level: at.level, + picked: failed.error.attempt().is_none(), + }); + break; + } + self.host.observe(&RouterEvent::Retry { + model_group: target.group.clone(), + retry: retry + 1, + delay, + }); + match self + .attempt( + snapshot, + op, + request, + target, + &inputs, + (retry + 1, num_retries), + place, + ) + .await + { + Ok(mut response) => { + response.max_retries = Some(num_retries); + return Ok(response); + } + Err(next) if next.error.attempt().is_some_and(|f| f.agentic_followup) => { + return Err(next); + } + Err(next) => { + self.pick_failed(at, target, &next.error); + failed = next; + } + } + } + Err(failed) + } + + /// `error` is a pick's: reported as it happens, whatever the level then does with it. + fn pick_failed(&self, at: FallbackHop, target: &Target, error: &Error) { + if let Some(picked) = PickError::of(error) { + self.host.observe(&RouterEvent::PickFailed { + level: at.level, + model: target.group.clone(), + error: picked, + }); + } + } + + /// The level serving `target` raises a pick's `error` ([`RouterEvent::LevelFailed`]); + /// `handling` when its fallback handling raised it. + fn level_failed(&self, at: FallbackHop, target: &Target, error: &Error, handling: bool) { + if let Some(picked) = PickError::of(error) { + self.host.observe(&RouterEvent::LevelFailed { + level: at.level, + model: target.group.clone(), + error: picked, + handling, + }); + } + } + + /// `_core_log_retry` for a failure in the level's retry loop. + fn log_retry(&self, target: &Target, at: FallbackHop, retry: u32, error: &Error) { + let cause = match RetryCause::of(error) { + RetryCause::Other => RetryCause::Attempt, + cause => cause, + }; + self.host.observe(&RouterEvent::RetryLogged { + scope: at.scope, + level: at.level, + model: target.group.clone(), + attempted_retries: Some(retry), + cause, + }); + } + + /// A `mock_testing_*` failure for the level's group, reported to the host, which raises + /// the matching exception. + fn mock_failed( + &self, + mock: MockFailure, + target: &Target, + at: FallbackHop, + num_retries: Option, + ) -> Error { + self.host.observe(&RouterEvent::MockFailed { + mock, + level: at.level, + model: target.group.clone(), + num_retries, + }); + Error::Attempt(mock.failure(num_retries)) + } + + fn should_retry( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + failure: &AttemptError, + healthy: usize, + all: usize, + ) -> bool { + let lists = Lists::of(snapshot, request); + should_retry( + failure, + RetryContext { + healthy, + all, + has_context_window_fallbacks: lists.context_window.is_some(), + has_content_policy_fallbacks: lists.content_policy.is_some(), + has_regular_fallbacks: lists.has_regular(), + }, + ) + } + + /// `_time_to_sleep_before_retry` and `calculate_retry_after`. + fn time_to_sleep( + &self, + snapshot: &RouterStateSnapshot, + retry_after: Option, + remaining_retries: u32, + num_retries: u32, + healthy: usize, + all: usize, + ) -> Duration { + if all != 1 && healthy > 0 { + return Duration::ZERO; + } + let jitter = JITTER * self.rng.next_f64(); + if let Some(retry_after) = retry_after + && !retry_after.is_zero() + && retry_after <= MAX_HONOURED_RETRY_AFTER + { + return retry_after + Duration::from_secs_f64(jitter); + } + let exponent = num_retries.saturating_sub(remaining_retries); + let backoff = (INITIAL_RETRY_DELAY * 2f64.powi(exponent as i32)) + .max(snapshot.settings.retry_after.as_secs_f64()) + .min(MAX_RETRY_DELAY); + Duration::from_secs_f64(backoff + jitter) + } + + /// One attempt: select a deployment, run it through the host, record the outcome. + #[allow(clippy::too_many_arguments)] + async fn attempt( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + inputs: &SelectInputs, + (retry, max_retries): (u32, u32), + (at, attempted): (FallbackHop, &[Tried]), + ) -> Result { + if request.unrouted { + return self + .unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at) + .await; + } + let pick_started = Instant::now(); + let (deployment, candidates) = match self + .picked(snapshot, op, request, target, inputs, retry, at) + .await + { + Ok(picked) => picked, + Err(failed) if request.passthrough_on_no_deployment => { + self.pick_failed(at, target, &failed.error); + return self + .unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at) + .await; + } + Err(failed) => return Err(failed), + }; + self.host.observe(&RouterEvent::Selected { + model_group: candidates.model_group.clone(), + deployment_id: deployment.id.clone(), + pick: pick_started.elapsed(), + }); + if !self.host.supports(op, deployment.provider()) { + return Err(Error::Config(ConfigError::Unsupported(Unsupported::Op { + op: op.clone(), + provider: deployment.provider().to_owned(), + })) + .into()); + } + // A name no deployment serves is routed to the first default fallback + // (`_common_checks_available_deployment`), still stamped as the name. + let defaulted = candidates.model_group != target.group + && matches!( + crate::resolve::resolve_request( + &snapshot.registry, + &target.group, + request.specific_deployment, + crate::resolve::Caller { + team_id: request.team_id.as_deref(), + proxy_admin: request.proxy_admin, + allowed_models: &request.allowed_models, + wildcard_request: candidates.wildcard_request.as_deref(), + ..crate::resolve::Caller::default() + }, + None, + snapshot.settings.dynamic.default_ref(), + jiff::Timestamp::now(), + ), + Ok(None) + ); + let attempt = Attempt { + op: op.clone(), + request: request.clone(), + deployment: deployment.clone(), + name: target.group.clone(), + model_group: if defaulted { + target.group.clone() + } else { + candidates.model_group.clone() + }, + target_order: target.order, + candidate_ids: candidates.all.iter().map(|d| d.id.clone()).collect(), + retry, + fallback_depth: at.depth, + fallback_level: at.level, + timeout: snapshot.settings.timeout, + params: target.params.clone(), + scope: at.scope, + max_retries, + model_group_size: snapshot.registry.model_list_len(&target.group), + }; + // Every attempt runs through the host; the core records its outcome itself, as far as the + // request's `OutcomeCallbacks` reach, except a deferred stream's: its request counts now + // (`increment_deployment_usage_for_response`), its success or failure when the host reports + // the stream's end. + match self.host.invoke(&attempt).await { + Ok(output) => { + let tracks = output + .tracks_usage + .unwrap_or_else(|| deployment.tracks_usage()); + let recorded = match (output.deferred, tracks) { + (true, true) => { + let tokens = output.total_tokens as f64; + self.health.add_usage(&deployment, None, tokens, 1.0).await + } + (true, false) => Ok(()), + (false, _) => { + self.health + .record_success( + request.outcome_callbacks, + &deployment, + output.total_tokens, + tracks, + ) + .await + } + }; + if let Err(error) = recorded { + warn!(deployment_id = %deployment.id, error = %error, "usage was not recorded"); + } + self.host.observe(&RouterEvent::Succeeded { + deployment_id: deployment.id.clone(), + }); + // `_should_raise_content_policy_error`: a blocked response is raised only when + // a content-policy fallback would serve it, and counts as no failure. + let raised = + match output.blocked { + Some(Blocked::ContentFilter) => self + .has_content_policy_fallback(snapshot, request, target, at, attempted), + Some(Blocked::Refusal) => self + .refusal_fallback_available(snapshot, request, target, at, attempted), + None => false, + }; + if let Some(blocked) = output.blocked.filter(|_| raised) { + self.host.observe(&RouterEvent::Blocked { + level: at.level, + model: target.group.clone(), + blocked, + }); + let mut failure = + AttemptError::new(FailureKind::ContentPolicyViolation, Some(400)); + failure.classes = crate::retry::content_policy_classes(); + return Err(Failed { + error: Box::new(Error::Attempt(failure)), + deployment_id: Some(deployment.id.clone()), + }); + } + Ok(RouterResponse { + output, + deployment_id: deployment.id.clone(), + model_group: candidates.model_group, + attempted_retries: retry, + max_retries: None, + attempted_fallbacks: at.depth, + }) + } + Err(failure) => { + if !failure.recorded { + self.record_failure(snapshot, request, &candidates, &deployment, &failure) + .await; + } + Err(Failed { + error: Box::new(Error::Attempt(failure)), + deployment_id: Some(deployment.id.clone()), + }) + } + } + } + + /// The attempt's pick: the deployment and the candidates it was picked from. + #[allow(clippy::too_many_arguments)] + async fn picked( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + inputs: &SelectInputs, + retry: u32, + at: FallbackHop, + ) -> Result<(Arc, Candidates), Failed> { + crate::routing_plugins::refuse_sync_pick( + snapshot.settings.routing_plugins, + &snapshot.settings.routing_strategy, + request, + )?; + let routed = self + .pre_routed(PreRoute { + op, + request, + name: &target.group, + fallback_depth: at.depth, + retry, + }) + .await?; + match routed { + PreRouted::Model(name) => { + snapshot.missing_routing_group(request, &name)?; + let candidates = self + .candidates(snapshot, op, request, &name, inputs) + .await?; + if candidates.healthy.is_empty() { + return Err(Error::NoHealthyDeployments { + model: candidates.failed_model(&snapshot.registry), + } + .into()); + } + let deployment = self.pick(snapshot, op, request, &candidates).await?; + Ok((deployment, candidates)) + } + // A custom strategy's pick is served as is, for the requested name. + PreRouted::Deployment(deployment) => { + let candidates = Candidates { + name: target.group.clone(), + model_group: target.group.clone(), + kind: ResolveKind::Group, + all: vec![deployment.clone()], + healthy: vec![deployment.clone()], + wildcard_request: None, + }; + Ok((deployment, candidates)) + } + } + } + + /// An attempt on a name that is no model group: the host serves it under that name, with no + /// deployment picked and no outcome recorded. + async fn unrouted_attempt( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + (retry, max_retries): (u32, u32), + at: FallbackHop, + ) -> Result { + let deployment = + Arc::new(crate::registry::Deployment::unrouted(&target.group).map_err(Error::from)?); + let attempt = Attempt { + op: op.clone(), + request: request.clone(), + deployment, + name: target.group.clone(), + model_group: target.group.clone(), + target_order: target.order, + candidate_ids: Vec::new(), + retry, + fallback_depth: at.depth, + fallback_level: at.level, + timeout: snapshot.settings.timeout, + params: target.params.clone(), + scope: at.scope, + max_retries, + model_group_size: snapshot.registry.model_list_len(&target.group), + }; + match self.host.invoke(&attempt).await { + Ok(output) => Ok(RouterResponse { + output, + deployment_id: String::new(), + model_group: target.group.clone(), + attempted_retries: retry, + max_retries: None, + attempted_fallbacks: at.depth, + }), + Err(failure) => Err(Error::Attempt(failure).into()), + } + } + + /// `_has_content_policy_fallback`: the content-policy chain resolves for the level's + /// lookup groups and holds a target the chain has not tried; with no content-policy list, + /// any router `"*"` entry. + fn has_content_policy_fallback( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + at: FallbackHop, + attempted: &[Tried], + ) -> bool { + let lists = Lists::of(snapshot, request); + match lists.content_policy { + Some(list) => unattempted_chain(list, &lists, request, target, at, attempted), + None => snapshot.settings.dynamic.fallbacks.has_star(), + } + } + + /// `_refusal_fallback_available`: the content-policy list decides when there is one; + /// else a router `"*"` entry, else a generic chain with a target the chain has not tried. + fn refusal_fallback_available( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + at: FallbackHop, + attempted: &[Tried], + ) -> bool { + if request.disable_fallbacks { + return false; + } + let lists = Lists::of(snapshot, request); + if lists.content_policy.is_some() { + return self.has_content_policy_fallback(snapshot, request, target, at, attempted); + } + if snapshot.settings.dynamic.fallbacks.has_star() { + return true; + } + lists + .fallbacks + .is_some_and(|list| unattempted_chain(list, &lists, request, target, at, attempted)) + } + + /// `async_deployment_callback_on_failure`, when it runs for this failure. + async fn count_failed_request( + &self, + deployment: &crate::registry::Deployment, + failure: &AttemptError, + ) { + if failure.uncounted_request { + return; + } + if let Err(error) = self.health.count_failed_request(deployment, None).await { + warn!(deployment_id = %deployment.id, error = %error, "usage was not recorded"); + } + } + + async fn record_failure( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + candidates: &Candidates, + deployment: &crate::registry::Deployment, + failure: &AttemptError, + ) { + self.host.observe(&RouterEvent::AttemptFailed { + deployment_id: deployment.id.clone(), + kind: failure.kind, + status: failure.status, + }); + let recorded = request.outcome_callbacks.record( + OutcomeWrite::Failure, + self.attempt_failure(snapshot, request, candidates, deployment, failure), + ); + if let Err(error) = recorded.await { + warn!(deployment_id = %deployment.id, error = %error, "cooldown was not recorded"); + } + } + + /// What the failure callbacks record for a failed attempt. + async fn attempt_failure( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + candidates: &Candidates, + deployment: &crate::registry::Deployment, + failure: &AttemptError, + ) -> Result<(), Error> { + // The default deployment's copy is no configured deployment: Python's cooldown check finds + // no group for its id and records nothing, unless the id names a configured deployment, + // which then takes the failure. + let configured; + let deployment = if candidates.kind == ResolveKind::Default { + match snapshot.registry.deployment(&deployment.id) { + Some(found) => { + configured = found; + &*configured + } + None => { + self.count_failed_request(deployment, failure).await; + return Ok(()); + } + } + } else { + deployment + }; + let shape = GroupShape { + // The deployment's own group decides, also when the call named the deployment by id. + single_deployment: snapshot + .registry + .group(&deployment.model_name) + .map_or(candidates.all.len() == 1, |group| group.len() == 1) + // A routing group call with alternatives cools its member down so selection + // moves on (`routing_group_alternatives`). + && !(candidates.kind == ResolveKind::RoutingGroup && candidates.all.len() > 1) + // As does a team deployment with a sibling under its public name + // (`team_model_alternatives`). + && !team_model_has_alternatives(&snapshot.registry, deployment), + has_fallback_group: snapshot + .settings + .dynamic + .fallback_targets(&[candidates.model_group.as_str()]) + .is_some(), + }; + self.count_failed_request(deployment, failure).await; + match self + .health + .record_failure( + &snapshot.settings, + deployment, + shape, + request.cooldown_time, + failure, + ) + .await + { + Ok(Some(cooldown)) => self.host.observe(&RouterEvent::CooldownSet { + deployment_id: deployment.id.clone(), + cooldown, + }), + Ok(None) => {} + Err(error) => return Err(error), + } + Ok(()) + } +} diff --git a/litellm-rust/crates/router-core/src/call/stream.rs b/litellm-rust/crates/router-core/src/call/stream.rs new file mode 100644 index 00000000000..a0fce4372dd --- /dev/null +++ b/litellm-rust/crates/router-core/src/call/stream.rs @@ -0,0 +1,456 @@ +//! Recovering a stream that failed after its attempt returned it. The host wraps every streamed +//! response; when the stream fails before the caller received content, the wrapper asks the core +//! what may still take over and hands the failure back here. Mirrors the decisions of the Python +//! router's stream wrappers: `_stream_can_fall_back`, `_refusal_fallback_available`, +//! `_anthropic_messages_stream_can_retry`, `_stream_retry_budget`, `_astream_retry_same_group`, +//! and the `async_function_with_fallbacks_common_utils` re-entry of the mid-stream fallback. + +use std::sync::Arc; + +use serde::Deserialize; + +use super::{Failed, FallbackChain, FallbackHop, Lists, Target, Tried, group_members}; +use crate::{ + Router, RouterStateSnapshot, + config::Op, + error::{AttemptError, Error, FailureKind}, + fallback::FallbackTarget, + health::should_retry_status, + host::{RouterEvent, RouterRequest, UncountedRetries}, + select::{Candidates, SelectInputs}, +}; + +/// Where a streamed response came from: the attempt that returned it. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct StreamOrigin { + /// The name the attempt served: the requested group, or the fallback target. + pub group: String, + /// The `order` level an order fallback targeted. + pub order: Option, + /// `fallback_depth` of the attempt. + pub depth: u32, + pub level: u32, + /// The deployment the attempt ran on. + pub deployment_id: Option, + /// Same-group stream retries this request already spent (`attempted_retries_for_request`). + pub attempted_retries: u32, + /// The budget the first stream retry committed to (`committed_retry_budget_for_request`). + pub committed_budget: Option, +} + +/// What may still take over a stream that fails before content. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StreamPolicy { + /// `_anthropic_messages_stream_can_retry`. + pub can_retry: bool, + /// `_stream_can_fall_back`. + pub can_fall_back: bool, + /// `_refusal_fallback_available`. + pub refusal_fallback: bool, + /// `_stream_retry_budget` for the failure kind asked about: the budget, and whether a retry + /// policy governs the retry. + pub retry_budget: Option<(u32, bool)>, +} + +impl StreamOrigin { + fn target(&self) -> Target { + Target { + group: self.group.clone(), + order: self.order, + params: None, + excluded: Arc::from([]), + failover_excluded: Arc::from([]), + } + } + + /// `fallback_lookup_groups`: the served group, then the group the request was for. + fn lookup<'a>(&'a self, request: &'a RouterRequest) -> Vec<&'a str> { + let mut lookup = vec![self.group.as_str()]; + if request.model != self.group { + lookup.push(request.model.as_str()); + } + lookup + } + + /// `has_unattempted_fallback_target`. The first attempt carries no attempted targets, so any + /// resolved chain counts; a fallback hop has already tried its own group and the original one. + fn has_unattempted(&self, request: &RouterRequest, chain: Option>) -> bool { + let Some(chain) = chain else { + return false; + }; + if self.depth == 0 { + return true; + } + chain + .iter() + .any(|target| *target != self.group && *target != request.model) + } +} + +/// The group names a chain targets (a dict target counts by its `model`). +fn names(chain: Option>) -> Option> { + chain.map(|targets| { + targets + .iter() + .filter_map(|target| target.model().map(str::to_owned)) + .collect() + }) +} + +/// The `content_policy_fallbacks` chain for the lookup groups: the request's list where it +/// passed one, else the router's (`_has_content_policy_fallback`). +fn content_policy_targets(lists: &Lists<'_>, lookup: &[&str]) -> Option>> { + lists + .content_policy + .map(|list| names(list.lookup_groups(lookup, lists.keys))) +} + +/// The router's `default_fallbacks`, which Python folds into its `fallbacks` as `{"*": [...]}`: a +/// request's own list replaces them. +fn folded_defaults<'a>(snapshot: &'a RouterStateSnapshot, request: &RouterRequest) -> &'a [String] { + match request.fallbacks { + Some(_) => &[], + None => &snapshot.settings.dynamic.default_fallbacks, + } +} + +/// The regular fallback targets for the lookup groups, then the folded `default_fallbacks`. +fn regular_targets( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + lists: &Lists<'_>, + lookup: &[&str], +) -> Option> { + let defaults = folded_defaults(snapshot, request); + names( + lists + .fallbacks + .and_then(|list| list.lookup_groups(lookup, lists.keys)), + ) + .or_else(|| (!defaults.is_empty()).then(|| defaults.to_vec())) +} + +/// `_has_default_fallbacks`: the router's list, whatever the request passed. +fn has_default_fallbacks(snapshot: &RouterStateSnapshot) -> bool { + let dynamic = &snapshot.settings.dynamic; + dynamic.fallbacks.has_star() || !dynamic.default_fallbacks.is_empty() +} + +impl Router { + /// What may still take over a stream from `origin` that fails before content, and the retry + /// budget for a failure of `kind`. + pub fn stream_policy( + &self, + request: &RouterRequest, + origin: &StreamOrigin, + kind: Option, + ) -> StreamPolicy { + let snapshot = self.core.snapshot.load_full(); + StreamPolicy { + can_retry: stream_can_retry(&snapshot, request, origin), + can_fall_back: stream_can_fall_back(&snapshot, request, origin), + refusal_fallback: refusal_fallback_available(&snapshot, request, origin), + retry_budget: kind.map(|kind| stream_retry_budget(&snapshot, request, origin, kind)), + } + } + + /// The fallback chain for a stream from `origin` that failed with `failure` before content: + /// `async_function_with_fallbacks_common_utils` re-entered from the stream wrapper, so the + /// group's own retry loop does not run again. + pub async fn fallback_stream( + &self, + op: Op, + request: RouterRequest, + origin: StreamOrigin, + failure: AttemptError, + ) -> Result { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + let request = Arc::new(request); + let mut chain = FallbackChain::default(); + if request.model != origin.group { + chain + .attempted + .push(Tried::Target(FallbackTarget::Name(request.model.clone()))); + } + self.core + .fall_back( + &snapshot, + &op, + &request, + origin.target(), + FallbackHop { + depth: origin.depth, + level: origin.level, + scope: 0, + }, + &mut chain, + Failed::from(Error::Attempt(failure)), + ) + .await + } + + /// `_astream_retry_same_group`: re-run the attempt in the stream's own group until a retry + /// returns a response or the budget runs out. The failed result carries the newest failure. + pub async fn retry_stream( + &self, + op: Op, + request: RouterRequest, + origin: StreamOrigin, + failure: AttemptError, + ) -> Result { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + let request = Arc::new(request); + let target = origin.target(); + let at = FallbackHop { + depth: origin.depth, + level: origin.level, + scope: 0, + }; + let counting = SelectInputs { + count_only: true, + ..SelectInputs::default() + }; + let counted = |c: &Candidates, n: usize| if c.kind.is_single() { 0 } else { n }; + let (healthy, all) = self + .core + .candidates(&snapshot, &op, &request, &target.group, &counting) + .await + .map(|c| (counted(&c, c.healthy.len()), counted(&c, c.all.len()))) + .unwrap_or((0, 0)); + let (budget, policy_applies) = + stream_retry_budget(&snapshot, &request, &origin, failure.kind); + let inputs = SelectInputs { + target_order: target.order, + requested: Some(target.group.clone()), + fallback_hop: at.level > 0, + ..SelectInputs::default() + }; + let mut last = failure; + for attempt in origin.attempted_retries..budget { + if last.agentic_followup { + break; + } + if !policy_applies + && !self + .core + .should_retry(&snapshot, &request, &last, healthy, all) + { + break; + } + let delay = self.core.time_to_sleep( + &snapshot, + last.retry_after, + budget - attempt, + budget, + healthy, + all, + ); + self.core.host.sleep(delay).await; + self.core.host.observe(&RouterEvent::Retry { + model_group: target.group.clone(), + retry: attempt + 1, + delay, + }); + match self + .core + .attempt( + &snapshot, + &op, + &request, + &target, + &inputs, + (attempt + 1, budget), + (at, &[]), + ) + .await + { + Ok(mut response) => { + response.attempted_retries = attempt + 1; + response.max_retries = Some(budget); + return Ok(response); + } + Err(failed) => match *failed.error { + // `_stream_fallback_error_for_raised`: a status the pre-stream retry rule + // would not retry ends the retries with that failure. + Error::Attempt(failure) => { + let stop = failure.status.is_some_and(|s| !should_retry_status(s)); + last = failure; + if stop { + break; + } + } + other => return Err(other), + }, + } + } + Err(Error::Attempt(last)) + } +} + +/// The request's `num_retries`, then the routed deployment's, then the router's +/// (`_anthropic_messages_plain_retry_budget`). +fn plain_retry_budget( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> u32 { + if let Some(retries) = request.num_retries { + return retries; + } + if request.uncounted_retries == Some(UncountedRetries::Negative) { + return 0; + } + let deployment = origin + .deployment_id + .as_deref() + .and_then(|id| snapshot.registry.deployment(id)); + let configured = deployment.and_then(|d| match d.raw_params.get("num_retries") { + Some(serde_json::Value::Number(n)) => n.as_u64().and_then(|n| u32::try_from(n).ok()), + Some(serde_json::Value::String(s)) if s.chars().all(|c| c.is_ascii_digit()) => { + s.parse().ok() + } + _ => None, + }); + configured.unwrap_or(snapshot.settings.num_retries) +} + +/// The retry policy for the request's model group (`_anthropic_messages_resolved_retry_policy`), +/// unless the request opted out with `num_retries=0`. A policy that does not read as one governs +/// nothing here, with no fall through to the router's `retry_policy`. +fn retry_policy<'a>( + snapshot: &'a RouterStateSnapshot, + request: &'a RouterRequest, + origin: &StreamOrigin, +) -> Option<&'a crate::config::RetryPolicy> { + if request.num_retries == Some(0) || request.retry_policy_raises.contains(&origin.group) { + return None; + } + let settings = &snapshot.settings; + crate::retry::resolve_retry_policy( + settings.retry_policy.as_ref(), + &origin.group, + Some( + request + .model_group_retry_policy + .as_ref() + .unwrap_or(&settings.model_group_retry_policy), + ), + ) +} + +fn stream_retry_budget( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, + kind: FailureKind, +) -> (u32, bool) { + let failure = AttemptError::new(kind, None); + let policy_retries = + retry_policy(snapshot, request, origin).and_then(|p| p.retries_for(&failure)); + if origin.attempted_retries > 0 + && let Some(committed) = origin.committed_budget + { + return (committed, policy_retries.is_some()); + } + match policy_retries { + Some(retries) => (retries, true), + None => (plain_retry_budget(snapshot, request, origin), false), + } +} + +fn stream_can_retry( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> bool { + let attempted = origin.attempted_retries; + if attempted > 0 + && let Some(committed) = origin.committed_budget + { + return committed > attempted; + } + let plain = plain_retry_budget(snapshot, request, origin); + // `_retry_policy_ceiling`: every field the policy dumps. + let ceiling = retry_policy(snapshot, request, origin).map_or(plain, |policy| { + let granted = [ + policy.BadRequestErrorRetries, + policy.AuthenticationErrorRetries, + policy.TimeoutErrorRetries, + policy.RateLimitErrorRetries, + policy.ContentPolicyViolationErrorRetries, + policy.InternalServerErrorRetries, + policy.ServiceUnavailableErrorRetries, + policy.NotFoundErrorRetries, + policy.DefaultRetries, + ]; + plain.max(granted.into_iter().flatten().max().unwrap_or(0)) + }); + ceiling > attempted +} + +fn stream_can_fall_back( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> bool { + if request.disable_fallbacks { + return false; + } + if snapshot.settings.enable_weighted_failover { + return true; + } + // `_stream_order_levels`, the levels the fallback walk sees for the group. + let listing_team = request.listing_team_id.as_deref(); + let mut orders: Vec = group_members(snapshot, &origin.group, listing_team) + .iter() + .filter(|d| d.listed_for_team(listing_team)) + .filter_map(|d| d.order()) + .collect(); + orders.sort_unstable(); + orders.dedup(); + if orders.len() > 1 { + let current = origin.order.unwrap_or(orders[0]); + if orders.iter().any(|order| *order > current) { + return true; + } + } + let lists = Lists::of(snapshot, request); + let lookup = origin.lookup(request); + if let Some(chain) = content_policy_targets(&lists, &lookup) + && origin.has_unattempted(request, chain) + { + return true; + } + if !lists.has_regular() && folded_defaults(snapshot, request).is_empty() { + return false; + } + if lists + .fallbacks + .is_some_and(|list| list.is_client_side(lists.keys)) + { + return true; + } + origin.has_unattempted(request, regular_targets(snapshot, request, &lists, &lookup)) +} + +fn refusal_fallback_available( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> bool { + if request.disable_fallbacks { + return false; + } + let lists = Lists::of(snapshot, request); + let lookup = origin.lookup(request); + if let Some(chain) = content_policy_targets(&lists, &lookup) { + return origin.has_unattempted(request, chain); + } + if has_default_fallbacks(snapshot) { + return true; + } + origin.has_unattempted(request, regular_targets(snapshot, request, &lists, &lookup)) +} diff --git a/litellm-rust/crates/router-core/src/config.rs b/litellm-rust/crates/router-core/src/config.rs new file mode 100644 index 00000000000..fb9b7af215f --- /dev/null +++ b/litellm-rust/crates/router-core/src/config.rs @@ -0,0 +1,355 @@ +//! What the router is told: [`RouterConfig`], fixed for the router's life, and +//! [`RouterDynamicConfig`], a versioned snapshot read once per call. The router never reads +//! process globals. + +use std::{collections::BTreeMap, time::Duration}; + +use serde::Deserialize; + +use crate::{ + error::AttemptError, + registry::{DeploymentSpec, RoutingGroupSpec}, +}; + +/// `DEFAULT_COOLDOWN_TIME_SECONDS` in `litellm/constants.py`. +pub const DEFAULT_COOLDOWN_TIME: Duration = Duration::from_secs(5); +/// `ROUTER_MAX_FALLBACKS` in `litellm/constants.py`. +pub const DEFAULT_MAX_FALLBACKS: u32 = 5; + +/// The router operation a call runs: the Python router's endpoint method name (`acompletion`, +/// `aembedding`, ...). The core runs every op alike; what differs per endpoint is the host's. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize)] +#[serde(transparent)] +pub struct Op(String); + +impl Op { + pub fn new(name: impl Into) -> Self { + Self(name.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From<&str> for Op { + fn from(name: &str) -> Self { + Self::new(name) + } +} + +impl std::fmt::Display for Op { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// `routing_strategy`. Only simple-shuffle is native; the rest go through `Host::select`. +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + strum::EnumString, + strum::Display, + serde_with::DeserializeFromStr, +)] +pub enum RoutingStrategy { + #[default] + #[strum(serialize = "simple-shuffle")] + SimpleShuffle, + #[strum(serialize = "least-busy")] + LeastBusy, + #[strum(serialize = "usage-based-routing")] + UsageBased, + #[strum(serialize = "usage-based-routing-v2")] + UsageBasedV2, + #[strum(serialize = "latency-based-routing")] + LatencyBased, + #[strum(serialize = "cost-based-routing")] + CostBased, + /// `routing_strategy=None` passed explicitly (sent as `"None"`): stored as None, so no + /// strategy arm matches and every pick finds no deployment. + #[strum(serialize = "None")] + Unset, + #[strum(default, transparent)] + Other(String), +} + +impl RoutingStrategy { + pub fn is_native(&self) -> bool { + matches!(self, Self::SimpleShuffle | Self::Unset) + } +} + +/// `PluginKind` in `litellm/router_api.py`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::Display)] +pub enum PluginKind { + #[serde(rename = "auto")] + #[strum(serialize = "auto")] + Auto, + #[serde(rename = "complexity")] + #[strum(serialize = "complexity")] + Complexity, + #[serde(rename = "quality")] + #[strum(serialize = "quality")] + Quality, + #[serde(rename = "adaptive")] + #[strum(serialize = "adaptive")] + Adaptive, + #[serde(rename = "routing")] + #[strum(serialize = "routing")] + Routing, + #[serde(rename = "budget_limiter")] + #[strum(serialize = "budget_limiter")] + BudgetLimiter, + #[serde(rename = "prompt_caching_pins")] + #[strum(serialize = "prompt_caching_pins")] + PromptCachingPins, +} + +/// One registered plugin. Its behaviour lives in the host until it is ported. +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct PluginSpec { + pub kind: PluginKind, + pub name: String, + #[serde(default)] + pub config: serde_json::Value, +} + +/// `RetryPolicy` in `litellm/types/router.py`: retries per exception type. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[allow(non_snake_case)] +pub struct RetryPolicy { + pub BadRequestErrorRetries: Option, + pub AuthenticationErrorRetries: Option, + pub TimeoutErrorRetries: Option, + pub RateLimitErrorRetries: Option, + pub ContentPolicyViolationErrorRetries: Option, + pub InternalServerErrorRetries: Option, + pub ServiceUnavailableErrorRetries: Option, + pub NotFoundErrorRetries: Option, + pub DefaultRetries: Option, +} + +/// `AllowedFailsPolicy` in `litellm/types/router.py`: allowed fails per minute per exception type. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[allow(non_snake_case)] +pub struct AllowedFailsPolicy { + pub BadRequestErrorAllowedFails: Option, + pub AuthenticationErrorAllowedFails: Option, + pub TimeoutErrorAllowedFails: Option, + pub RateLimitErrorAllowedFails: Option, + pub ContentPolicyViolationErrorAllowedFails: Option, + pub InternalServerErrorAllowedFails: Option, + pub ServiceUnavailableErrorAllowedFails: Option, + pub BadGatewayErrorAllowedFails: Option, + pub NotFoundErrorAllowedFails: Option, +} + +impl AllowedFailsPolicy { + /// `_core_get_allowed_fails_from_policy`: the first set field whose exception class the + /// failure is an instance of, in Python's order. + pub fn allowed_fails_for(&self, failure: &AttemptError) -> Option { + [ + ("AuthenticationError", self.AuthenticationErrorAllowedFails), + ("Timeout", self.TimeoutErrorAllowedFails), + ("RateLimitError", self.RateLimitErrorAllowedFails), + ( + "ContentPolicyViolationError", + self.ContentPolicyViolationErrorAllowedFails, + ), + ("BadRequestError", self.BadRequestErrorAllowedFails), + ("InternalServerError", self.InternalServerErrorAllowedFails), + ( + "ServiceUnavailableError", + self.ServiceUnavailableErrorAllowedFails, + ), + ("BadGatewayError", self.BadGatewayErrorAllowedFails), + ("NotFoundError", self.NotFoundErrorAllowedFails), + ] + .into_iter() + .find_map(|(class, allowed)| allowed.filter(|_| failure.is_a(class))) + } +} + +pub use crate::fallback::{FallbackKeys, Fallbacks}; + +/// Static config, fixed for the router's life. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct RouterConfig { + pub model_list: Vec, + pub routing_strategy: RoutingStrategy, + pub routing_groups: Vec, + /// `_model_to_group` as Python last built it (member model group to routing group), set + /// only when a raw `routing_groups` write replaced the groups since: member calls keep + /// routing by it, as the Python router's stale map does. + pub routing_group_members: Option>, + pub plugins: Vec, + /// Router default `cooldown_time`; `None` is `DEFAULT_COOLDOWN_TIME_SECONDS`. + pub cooldown_time: Option, + pub disable_cooldowns: bool, + pub allowed_fails_policy: Option, + pub retry_policy: Option, + /// `model_group_retry_policy`: a group's own policy, which wins over `retry_policy`. + pub model_group_retry_policy: BTreeMap, + /// `retry_after`: the minimum backoff between retries, in seconds. + pub retry_after: f64, + pub timeout: Option, + pub request_timeout: Option, + pub drop_params: Option, + pub enable_pre_call_checks: bool, + /// `Router(plugins=[...])` is set: the host runs the routing-plugin pipeline. + pub routing_plugins: bool, + /// The host's `CustomLogger` deployment filters run on every async candidate list + /// (`async_callback_filter_deployments`). + pub callback_filters: bool, + pub enable_health_check_routing: bool, + /// How old (seconds) a background health check result may be and still count + /// (`health_check_staleness_threshold`, else the default interval times its multiplier). + pub health_check_staleness_threshold: Option, + /// `background_health_check_model_groups`: when set, only these groups are health-filtered. + pub background_health_check_model_groups: Option>, + /// `enable_weighted_failover`: a failed deployment's group is re-picked without it before + /// any cross-group fallback (simple-shuffle only). + pub enable_weighted_failover: bool, + /// `fallback_on_unsupported_params`; `None` is Python's default, `True`. + pub fallback_on_unsupported_params: Option, + /// Deployment ids that are a provider's default and never cool down + /// (`provider_default_deployment_ids`). + pub provider_default_deployment_ids: Vec, + /// The operations this router must serve: `Host::supports` must accept each one for every + /// configured provider, or the router fails to build. + pub served_ops: Vec, +} + +/// Dynamic config: a versioned snapshot read once per call. +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct RouterDynamicConfig { + pub version: u64, + /// `model_group_alias` as Python spells it: each alias maps to a group name, or to + /// `{"model": , "hidden": }`. + #[serde(alias = "model_group_alias", deserialize_with = "alias_targets")] + pub model_alias_map: BTreeMap, + pub fallbacks: Fallbacks, + pub default_fallbacks: Vec, + pub context_window_fallbacks: Option, + pub content_policy_fallbacks: Option, + pub num_retries: Option, + pub max_fallbacks: u32, + /// `allowed_fails`, as the router copied it when it was built. + pub allowed_fails: Option, + /// `allowed_fails` is only the process default: it was not passed to the router and still + /// equals `litellm.allowed_fails`. Anything else switches cooldowns from the failure-ratio + /// rule to the allowed-fails counter (`_is_allowed_fails_set_on_router`). + pub allowed_fails_is_default: bool, + /// Other spellings a fallback key matches. + pub fallback_keys: FallbackKeys, + /// `default_deployment`: what a name no deployment, alias, routing group, team route or + /// wildcard serves is routed to, as a copy whose `litellm_params.model` is the name + /// (`_try_early_resolve_deployments_for_model_not_in_names`). + pub default_deployment: Option, + /// The `default_deployment` is a value Python cannot copy for a name: resolving to it + /// raises the copy's error ([`crate::Error::DefaultDeploymentRaises`]). + pub default_deployment_raises: bool, +} + +/// `model_group_alias` as `resolve_model_group_alias` reads it: an alias maps to a non-empty +/// group name, or to `{"model": , ...}`; any other value rewrites nothing. +fn alias_targets<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let aliases = BTreeMap::::deserialize(deserializer)?; + Ok(aliases + .into_iter() + .filter_map(|(alias, target)| { + let group = match &target { + serde_json::Value::Object(entry) => entry.get("model")?, + other => other, + }; + group + .as_str() + .filter(|group| !group.is_empty()) + .map(|group| (alias, group.to_owned())) + }) + .collect()) +} + +impl Default for RouterDynamicConfig { + fn default() -> Self { + Self { + version: 0, + model_alias_map: BTreeMap::new(), + fallbacks: Fallbacks::default(), + default_fallbacks: Vec::new(), + context_window_fallbacks: None, + content_policy_fallbacks: None, + num_retries: None, + max_fallbacks: DEFAULT_MAX_FALLBACKS, + allowed_fails: None, + allowed_fails_is_default: false, + fallback_keys: FallbackKeys::default(), + default_deployment: None, + default_deployment_raises: false, + } + } +} + +impl RouterDynamicConfig { + /// The `default_deployment` as resolution serves it. + pub fn default_ref(&self) -> Option> { + if self.default_deployment_raises { + return Some(crate::resolve::DefaultDeployment::Raises); + } + self.default_deployment + .as_ref() + .map(crate::resolve::DefaultDeployment::Spec) + } + + /// The regular fallback targets for the lookup groups, then `default_fallbacks`, which + /// Python folds into `fallbacks` as `{"*": [...]}`: what `_has_fallback_group` reads. + pub fn fallback_targets(&self, groups: &[&str]) -> Option> { + if groups + .iter() + .any(|group| self.fallbacks.has_keyed_chain(group, &self.fallback_keys)) + { + return Some(Vec::new()); + } + (!self.default_fallbacks.is_empty()).then(|| self.default_fallbacks.clone()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::group_name(json!({"model_group_alias": {"gpt": "gpt-4"}}))] + #[case::hidden_entry(json!({"model_group_alias": {"gpt": {"model": "gpt-4", "hidden": true}}}))] + #[case::native_name(json!({"model_alias_map": {"gpt": "gpt-4"}}))] + fn model_group_alias_reads_both_python_spellings(#[case] wire: serde_json::Value) { + let dynamic: RouterDynamicConfig = serde_json::from_value(wire).unwrap(); + assert_eq!( + dynamic.model_alias_map, + BTreeMap::from([("gpt".to_owned(), "gpt-4".to_owned())]) + ); + } + + #[rstest] + #[case::native("simple-shuffle", RoutingStrategy::SimpleShuffle)] + #[case::unknown("my-strategy", RoutingStrategy::Other("my-strategy".to_owned()))] + fn routing_strategy_parses_its_wire_name( + #[case] wire: &str, + #[case] expected: RoutingStrategy, + ) { + assert_eq!(wire.parse::().unwrap(), expected); + assert_eq!(expected.to_string(), wire); + } +} diff --git a/litellm-rust/crates/router-core/src/eligibility.rs b/litellm-rust/crates/router-core/src/eligibility.rs new file mode 100644 index 00000000000..5cedf218b9a --- /dev/null +++ b/litellm-rust/crates/router-core/src/eligibility.rs @@ -0,0 +1,240 @@ +//! Which deployments may serve a request's endpoint: a port of +//! `litellm/router_utils/endpoint_eligibility.py`. A deployment that sets +//! `enforce_supported_endpoints` and lists none of the chat-shaped endpoints serves none of them; +//! the host names the endpoint for the calls `endpoint_selection` wraps. + +use std::{collections::BTreeSet, sync::Arc}; + +use serde_json::{Map, Value}; + +use crate::{error::Error, registry::Deployment, resolve::Caller}; + +/// `CHAT_ENDPOINTS`: `_endpoint_path` of each value of `CHAT_ENDPOINT_BY_CALL`. +const CHAT_ENDPOINTS: [&str; 8] = [ + "/chat/completions", + "/completions", + "/messages", + "/responses", + "/generatecontent", + "/streamgeneratecontent", + "/interactions", + "/responses/compact", +]; + +/// `_endpoint_path`. +fn endpoint_path(endpoint: &str) -> String { + let rooted = format!("/{}", endpoint.trim().trim_matches('/').to_lowercase()); + let path = rooted.strip_prefix("/openai").unwrap_or(&rooted); + let path = path.strip_prefix("/v1beta").unwrap_or(path); + let path = path.strip_prefix("/v1").unwrap_or(path); + match path.rsplit_once(':') { + Some((_, action)) => format!("/{action}"), + None => path.to_owned(), + } +} + +/// The flag as pydantic's lax `bool | None` reads it: `None` for a value it refuses, which fails +/// the whole declaration. +fn enforce_flag(value: Option<&Value>) -> Option { + match value { + None | Some(Value::Null) => Some(false), + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::Number(number)) => match number.as_f64() { + Some(0.0) => Some(false), + Some(1.0) => Some(true), + _ => None, + }, + Some(Value::String(text)) => match text.to_lowercase().as_str() { + "0" | "off" | "f" | "false" | "n" | "no" => Some(false), + "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), + _ => None, + }, + Some(_) => None, + } +} + +/// `_enforced_endpoints`: the declared endpoints of a deployment that enforces them. One endpoint +/// may be written as a string; a declaration that does not validate enforces nothing. +fn enforced_endpoints(model_info: &Map) -> Option> { + if !enforce_flag(model_info.get("enforce_supported_endpoints"))? { + return None; + } + match model_info.get("supported_endpoints")? { + Value::String(endpoint) => Some(vec![endpoint.clone()]), + Value::Array(endpoints) => endpoints + .iter() + .map(|endpoint| endpoint.as_str().map(str::to_owned)) + .collect(), + _ => None, + } +} + +/// `non_chat_endpoints`: what a deployment serves when it serves no chat-shaped endpoint, else +/// `None`. +fn non_chat_endpoints(deployment: &Deployment) -> Option> { + let declared = enforced_endpoints(&deployment.model_info.raw)?; + let chat = declared + .iter() + .any(|endpoint| CHAT_ENDPOINTS.contains(&endpoint_path(endpoint).as_str())); + (!chat).then(|| declared.into_iter().collect()) +} + +/// `eligible_deployments`: the deployments that may serve the caller's endpoint. When none may, +/// the refusal names what they do serve. +pub(crate) fn eligible_deployments( + model: &str, + caller: Caller<'_>, + deployments: Vec>, +) -> Result>, Error> { + let Some(endpoint) = caller.endpoint.filter(|_| !deployments.is_empty()) else { + return Ok(deployments); + }; + let declared: Vec>> = deployments + .iter() + .map(|deployment| non_chat_endpoints(deployment)) + .collect(); + let eligible: Vec> = deployments + .into_iter() + .zip(&declared) + .filter(|(_, endpoints)| endpoints.is_none()) + .map(|(deployment, _)| deployment) + .collect(); + if !eligible.is_empty() { + return Ok(eligible); + } + let supported: BTreeSet = declared.into_iter().flatten().flatten().collect(); + let supported = if supported.is_empty() { + "none".to_owned() + } else { + Vec::from_iter(supported).join(", ") + }; + let requested = if caller.requested.is_empty() { + model + } else { + caller.requested + }; + Err(Error::EndpointNotSupported { + model: model.to_owned(), + message: format!( + "{endpoint} is not supported by model={requested}. Supported endpoints: {supported}" + ), + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn deployment(id: &str, info: Value) -> Arc { + let mut model_info = info; + model_info["id"] = json!(id); + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": model_info, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + fn chat<'a>() -> Caller<'a> { + Caller { + endpoint: Some("/v1/chat/completions"), + requested: "g", + ..Caller::default() + } + } + + #[rstest] + #[case::v1_prefix("/v1/chat/completions", "/chat/completions")] + #[case::openai_prefix("/openai/v1/responses/", "/responses")] + #[case::bare_gemini_action("generateContent", "/generatecontent")] + #[case::beta_prefix("/v1beta/interactions", "/interactions")] + #[case::gemini_model_path( + "/v1beta/models/gemini:streamGenerateContent", + "/streamgeneratecontent" + )] + fn endpoint_paths_normalize(#[case] endpoint: &str, #[case] expected: &str) { + assert_eq!(endpoint_path(endpoint), expected); + } + + #[rstest] + #[case::embeddings_only( + json!({"enforce_supported_endpoints": true, "supported_endpoints": ["/v1/embeddings"]}), + false + )] + #[case::one_endpoint_as_a_string( + json!({"enforce_supported_endpoints": true, "supported_endpoints": "/v1/embeddings"}), + false + )] + #[case::string_flag( + json!({"enforce_supported_endpoints": "yes", "supported_endpoints": ["/v1/embeddings"]}), + false + )] + #[case::an_empty_list_serves_nothing( + json!({"enforce_supported_endpoints": true, "supported_endpoints": []}), + false + )] + #[case::any_chat_shaped_endpoint_serves_every_one( + json!({"enforce_supported_endpoints": true, "supported_endpoints": ["/v1/responses"]}), + true + )] + #[case::not_enforced( + json!({"enforce_supported_endpoints": false, "supported_endpoints": ["/v1/embeddings"]}), + true + )] + #[case::unset_flag(json!({"supported_endpoints": ["/v1/embeddings"]}), true)] + #[case::enforced_without_a_list(json!({"enforce_supported_endpoints": true}), true)] + #[case::an_invalid_flag_enforces_nothing( + json!({"enforce_supported_endpoints": "maybe", "supported_endpoints": ["/v1/embeddings"]}), + true + )] + #[case::an_invalid_list_enforces_nothing( + json!({"enforce_supported_endpoints": true, "supported_endpoints": [1]}), + true + )] + fn a_declaration_keeps_or_drops_a_deployment(#[case] info: Value, #[case] kept: bool) { + let open = deployment("open", json!({})); + let declared = deployment("declared", info); + let eligible = eligible_deployments("g", chat(), vec![declared, open]).unwrap(); + assert_eq!(eligible.len(), if kept { 2 } else { 1 }); + } + + #[test] + fn no_endpoint_keeps_everything() { + let declared = deployment( + "d", + json!({"enforce_supported_endpoints": true, "supported_endpoints": []}), + ); + let eligible = eligible_deployments("g", Caller::default(), vec![declared]).unwrap(); + assert_eq!(eligible.len(), 1); + } + + #[rstest] + #[case::lists_what_they_serve( + json!(["/v1/rerank", "/v1/embeddings"]), + "/v1/chat/completions is not supported by model=g. \ + Supported endpoints: /v1/embeddings, /v1/rerank" + )] + #[case::nothing_declared( + json!([]), + "/v1/chat/completions is not supported by model=g. Supported endpoints: none" + )] + fn every_deployment_refusing_names_the_endpoint( + #[case] endpoints: Value, + #[case] expected: &str, + ) { + let declared = deployment( + "d", + json!({"enforce_supported_endpoints": true, "supported_endpoints": endpoints}), + ); + let refused = eligible_deployments("resolved", chat(), vec![declared]); + assert!(matches!( + refused, + Err(Error::EndpointNotSupported { model, message }) + if model == "resolved" && message == expected + )); + } +} diff --git a/litellm-rust/crates/router-core/src/error.rs b/litellm-rust/crates/router-core/src/error.rs new file mode 100644 index 00000000000..b4112501c36 --- /dev/null +++ b/litellm-rust/crates/router-core/src/error.rs @@ -0,0 +1,473 @@ +//! The router's errors: [`ConfigError`] when a router is built or changed, [`Error`] when a call +//! or a selection fails, and [`AttemptError`], the typed facts of one failed attempt. + +use std::{any::Any, fmt, sync::Arc, time::Duration}; + +use crate::config::{Op, RoutingStrategy}; + +/// What a host cannot serve, found when the router is built rather than partway through a call. +#[derive(Clone, Debug, PartialEq, thiserror::Error)] +pub enum Unsupported { + #[error("routing strategy `{0}` needs a host that can select")] + Strategy(RoutingStrategy), + #[error("routing plugin `{0}` needs a host that can select")] + Plugin(String), + #[error("the host cannot run operation `{op}` on provider `{provider}`")] + Op { op: Op, provider: String }, +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("the router config asks for something this host cannot serve")] + Unsupported(#[from] Unsupported), + #[error("deployment `{model_name}` has invalid litellm_params")] + InvalidDeployment { + model_name: String, + #[source] + source: serde_json::Error, + }, + #[error("deployment id `{id}` is configured more than once")] + DuplicateId { id: String }, + /// A removal the host's indexes could not finish ([`crate::host_index::RemovalError`]). + #[error("{0}")] + Removal(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("the router config is invalid")] + Config(#[from] ConfigError), + #[error("no deployment serves model `{model}`")] + UnknownModel { model: String }, + #[error("no deployment of `{model}` is available")] + NoHealthyDeployments { model: String }, + #[error("the attempt failed")] + Attempt(#[from] AttemptError), + #[error("the routing cache failed")] + Cache(#[from] litellm_cache::Error), + #[error("the router was retired")] + Retired, + /// A blocked deployment named by its id (`model` is its litellm model). + #[error("deployment model `{model}` is paused")] + Paused { model: String }, + /// A pass-through pick named a deployment id without `use_in_pass_through`. + #[error("deployment `{id}` does not support pass-through")] + NotPassThrough { id: String }, + /// A pass-through pick of a group with healthy deployments, none of them pass-through. + #[error("model `{model}` has no pass-through deployment")] + NoPassThroughDeployments { model: String }, + /// Every deployment `model` resolved to is a pre-routing strategy marker (`auto_router/...`). + #[error("every deployment of `{model}` is a strategy router marker")] + OnlyMarkers { model: String }, + /// Every deployment `model` resolved to refuses the request's endpoint + /// (`eligible_deployments`); `message` names what they serve. + #[error("{message}")] + EndpointNotSupported { model: String, message: String }, + /// Every deployment `model` resolved to is split into PTU shares the caller holds none of. + #[error("deployment `{model}` is reserved for the teams holding a PTU share of it")] + PtuReserved { model: String }, + /// A proxy admin without a team named a public model name several teams use. + #[error("model name `{model}` matches deployments from multiple teams")] + MultipleTeams { model: String }, + /// Every deployment `model` resolved to is reserved for other teams by an active access + /// window, the first of which ends at `until` (`HH:MM`) in `timezone`. + #[error("deployment `{model}` is reserved for another team until {until} {timezone}")] + Reserved { + model: String, + until: String, + timezone: String, + }, + /// The pre-call checks found no deployment whose context window fits the prompt; `detail` + /// names each one dropped for it. + #[error("no deployment of `{model}` has a context window large enough for this call")] + ContextWindowExceeded { model: String, detail: String }, + /// The pre-call checks found no deployment of `model` that takes the requested `params`. + #[error("no deployment of `{model}` supports {params:?}")] + UnsupportedParams { model: String, params: Vec }, + /// The routing plugins left none of the healthy deployments' models. + #[error("no deployments left after routing-plugin filtering: {models:?}")] + RoutingPluginsLeftNothing { models: Vec }, + /// The request's `_router_weights` failed `validate_router_weights` when simple-shuffle picked. + #[error("the request's router weights are invalid")] + InvalidRouterWeights, + /// A deployment's `litellm_params` `metric` (weight, rpm or tpm) is not a number. + #[error("deployment {metric} must be numeric")] + NonNumericWeight { metric: &'static str }, + /// A router with routing plugins was asked for a pick on the sync path, which never runs them. + #[error("routing plugins only run on the async selection path")] + RoutingPluginsNeedAsync, + /// The stale routing-group member map (`_model_to_group`, kept as it was when a raw + /// `routing_groups` write replaced the groups) names a group that is no longer configured: + /// Python's `_get_routing_context` raises `KeyError(group)` for every pick of the name. + #[error("routing group `{group}` is no longer configured")] + MissingRoutingGroup { group: String }, + /// The router's `default_deployment` is not a value Python can copy for `model` + /// (`_try_early_resolve_deployments_for_model_not_in_names` raises the copy's error). + #[error("the default deployment cannot be served for `{model}`")] + DefaultDeploymentRaises { model: String }, + /// A written model-name or team index names a position past the end of the model list: + /// Python's lookup of `model` raises `IndexError`. + #[error("a written index of `{model}` names a position past the model list")] + IndexOutOfRange { model: String }, + /// A model-info query raised where the Python router raises: `exception` names the Python + /// exception class (`KeyError`, `TypeError`, `AttributeError`, `ValueError`) and `detail` its + /// argument. + #[error("the model-info query raised {exception}({detail:?})")] + ModelInfo { + exception: &'static str, + detail: String, + }, +} + +impl Error { + /// The failed attempt behind this error, when a deployment was called. + pub fn attempt(&self) -> Option<&AttemptError> { + match self { + Self::Attempt(attempt) => Some(attempt), + Self::Config(_) + | Self::UnknownModel { .. } + | Self::NoHealthyDeployments { .. } + | Self::Cache(_) + | Self::Retired + | Self::Paused { .. } + | Self::NotPassThrough { .. } + | Self::NoPassThroughDeployments { .. } + | Self::OnlyMarkers { .. } + | Self::EndpointNotSupported { .. } + | Self::PtuReserved { .. } + | Self::MultipleTeams { .. } + | Self::Reserved { .. } + | Self::ContextWindowExceeded { .. } + | Self::UnsupportedParams { .. } + | Self::RoutingPluginsLeftNothing { .. } + | Self::InvalidRouterWeights + | Self::NonNumericWeight { .. } + | Self::RoutingPluginsNeedAsync + | Self::MissingRoutingGroup { .. } + | Self::DefaultDeploymentRaises { .. } + | Self::IndexOutOfRange { .. } + | Self::ModelInfo { .. } => None, + } + } + + /// The Python exception a pick that failed with this error raises, as the retry loop + /// decides by it (its class names, most specific first, and its status): `None` for an + /// attempt's failure and for errors no pick raises. + pub fn pick_failure(&self) -> Option { + const BAD_REQUEST: &[&str] = &["BadRequestError"]; + let (classes, status): (&[&str], Option) = match self { + Self::UnknownModel { .. } + | Self::OnlyMarkers { .. } + | Self::PtuReserved { .. } + | Self::NotPassThrough { .. } + | Self::NoPassThroughDeployments { .. } + | Self::MultipleTeams { .. } + | Self::Reserved { .. } => (BAD_REQUEST, Some(400)), + Self::EndpointNotSupported { .. } => ( + &["EndpointNotSupportedByModelError", "BadRequestError"], + Some(400), + ), + Self::ContextWindowExceeded { .. } => ( + &["ContextWindowExceededError", "BadRequestError"], + Some(400), + ), + Self::UnsupportedParams { .. } => { + (&["UnsupportedParamsError", "BadRequestError"], Some(400)) + } + Self::Paused { .. } => (&["ServiceUnavailableError"], Some(503)), + Self::NoHealthyDeployments { .. } => (&["RouterRateLimitError", "ValueError"], None), + Self::RoutingPluginsLeftNothing { .. } | Self::RoutingPluginsNeedAsync => { + (&["ValueError"], None) + } + Self::InvalidRouterWeights => (&["ValidationError", "ValueError"], None), + Self::NonNumericWeight { .. } => (&["TypeError"], None), + Self::MissingRoutingGroup { .. } => (&["KeyError", "LookupError"], None), + Self::IndexOutOfRange { .. } => (&["IndexError", "LookupError"], None), + Self::DefaultDeploymentRaises { .. } => (&["Exception"], None), + Self::Attempt(_) + | Self::Config(_) + | Self::Cache(_) + | Self::Retired + | Self::ModelInfo { .. } => return None, + }; + let mut failure = AttemptError::host_raised(classes); + failure.status = status; + Some(failure) + } + + /// A copy of a pick's error, for the events that name it; `None` for any other error. + pub fn pick_copy(&self) -> Option { + Some(match self { + Self::UnknownModel { model } => Self::UnknownModel { + model: model.clone(), + }, + Self::NoHealthyDeployments { model } => Self::NoHealthyDeployments { + model: model.clone(), + }, + Self::Paused { model } => Self::Paused { + model: model.clone(), + }, + Self::NotPassThrough { id } => Self::NotPassThrough { id: id.clone() }, + Self::NoPassThroughDeployments { model } => Self::NoPassThroughDeployments { + model: model.clone(), + }, + Self::OnlyMarkers { model } => Self::OnlyMarkers { + model: model.clone(), + }, + Self::EndpointNotSupported { model, message } => Self::EndpointNotSupported { + model: model.clone(), + message: message.clone(), + }, + Self::PtuReserved { model } => Self::PtuReserved { + model: model.clone(), + }, + Self::MultipleTeams { model } => Self::MultipleTeams { + model: model.clone(), + }, + Self::Reserved { + model, + until, + timezone, + } => Self::Reserved { + model: model.clone(), + until: until.clone(), + timezone: timezone.clone(), + }, + Self::ContextWindowExceeded { model, detail } => Self::ContextWindowExceeded { + model: model.clone(), + detail: detail.clone(), + }, + Self::UnsupportedParams { model, params } => Self::UnsupportedParams { + model: model.clone(), + params: params.clone(), + }, + Self::RoutingPluginsLeftNothing { models } => Self::RoutingPluginsLeftNothing { + models: models.clone(), + }, + Self::InvalidRouterWeights => Self::InvalidRouterWeights, + Self::NonNumericWeight { metric } => Self::NonNumericWeight { metric }, + Self::RoutingPluginsNeedAsync => Self::RoutingPluginsNeedAsync, + Self::MissingRoutingGroup { group } => Self::MissingRoutingGroup { + group: group.clone(), + }, + Self::DefaultDeploymentRaises { model } => Self::DefaultDeploymentRaises { + model: model.clone(), + }, + Self::IndexOutOfRange { model } => Self::IndexOutOfRange { + model: model.clone(), + }, + Self::Attempt(_) + | Self::Config(_) + | Self::Cache(_) + | Self::Retired + | Self::ModelInfo { .. } => return None, + }) + } +} + +/// The class of a failed attempt, as litellm's exception mapping names it. +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + strum::Display, + strum::EnumString, + strum::IntoStaticStr, + strum::VariantArray, +)] +pub enum FailureKind { + #[strum(serialize = "RateLimitError")] + RateLimit, + #[strum(serialize = "ContextWindowExceededError")] + ContextWindowExceeded, + #[strum(serialize = "ContentPolicyViolationError")] + ContentPolicyViolation, + #[strum(serialize = "AuthenticationError")] + Authentication, + #[strum(serialize = "PermissionDeniedError")] + PermissionDenied, + #[strum(serialize = "NotFoundError")] + NotFound, + #[strum(serialize = "BadRequestError")] + BadRequest, + #[strum(serialize = "Timeout")] + Timeout, + #[strum(serialize = "APIConnectionError")] + ApiConnection, + #[strum(serialize = "InternalServerError")] + InternalServer, + #[strum(serialize = "ServiceUnavailableError")] + ServiceUnavailable, + #[strum(serialize = "BadGatewayError")] + BadGateway, + /// A guardrail blocked the request: never retried, never a fallback. + #[strum(serialize = "GuardrailIntervention")] + GuardrailIntervention, + /// litellm rejected the request before calling the deployment (`MalformedRequestFieldError`, + /// `UnsupportedEndpointError`): never cools the deployment down. + #[strum(serialize = "Declined")] + Declined, + /// The caller's own stored credential was rejected, not the deployment's + /// (`CallerCredentialAuthenticationError`): never cools the deployment down. + #[strum(serialize = "CallerCredentialAuthenticationError")] + CallerCredentialAuthentication, + /// A per-user credential exchange was rate limited (`CallerCredentialRateLimitError`): never + /// cools the deployment down. + #[strum(serialize = "CallerCredentialRateLimitError")] + CallerCredentialRateLimit, + #[strum(serialize = "APIError")] + Other, +} + +/// An opaque value the host hands through the router unchanged, such as the original Python +/// exception or response object. +#[derive(Clone)] +pub struct Payload(pub Arc); + +impl fmt::Debug for Payload { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("Payload(..)") + } +} + +/// One failed attempt, as typed facts the retry, fallback and cooldown rules decide by. +#[derive(Clone, Debug, thiserror::Error)] +#[error("the deployment call failed with {kind}")] +pub struct AttemptError { + pub kind: FailureKind, + pub status: Option, + /// The provider's `retry-after` header (`_get_retry_after_from_exception_header`). + pub retry_after: Option, + /// A deployment-level `num_retries` the failure carries (Python's `exception.num_retries`). + pub num_retries: Option, + /// The deployment the exception itself names (`_deployment_ids_to_skip_on_retry` reads its + /// `retry_skip_deployment_id`, then `failed_deployment_id`): an exception raised again keeps + /// the id its first failure stamped, whichever deployment raised it now. + pub failed_deployment_id: Option, + /// The provider's error text, kept only for the cooldown record Python writes + /// (`exception_received`, masked past 50 characters). + pub detail: Option>, + /// The exception's class names, most specific first (its `__mro__`), for the rules Python + /// decides by `isinstance`. Empty when the host reports none; the kind then stands in. + pub classes: Vec>, + /// The host already recorded what this failure does to the deployment's health (a routing + /// pre-call check's failure, which Python's attempt handles itself: a rate limit cools the + /// deployment down, anything else records nothing), or no failure callback runs for it (it + /// was raised outside the litellm call, or a shared logging object already logged a failure), + /// so the core records nothing. + pub recorded: bool, + /// The exception names its provider (`llm_provider` is set), which marks a provider's own + /// 400 for `fallback_on_unsupported_params`. + pub names_provider: bool, + /// An agentic follow-up call failed after the first call was served + /// (`is_agentic_followup_failure`): never retried, since a retry reruns every paid round. The + /// failure keeps its own class, which decides its fallbacks and cooldown as for any failure. + pub agentic_followup: bool, + /// One of `deployment_callback_on_failure`'s early exits holds: the failure says nothing about + /// the deployment, so it is neither counted nor cooled down (a failed request still counts + /// against the minute's requests, which `async_deployment_callback_on_failure` does apart). + pub outside_deployment: bool, + /// One of `_should_run_cooldown_logic`'s per-caller exits holds (`_is_per_user_provider_request` + /// or a per-user Copilot deployment): the failure is counted but never cools the deployment. + pub caller_scoped: bool, + /// The exception has no `status_code` (Python's status `""`): `_is_cooldown_required` holds + /// no cooldown for it, unlike a status the host could not read, which cools down as a 500. + pub status_absent: bool, + /// The failure counts against no per-minute requests: `async_deployment_callback_on_failure` + /// does not run for a sync call, and returns at once for a batch retrieve. + pub uncounted_request: bool, + pub payload: Option, +} + +impl AttemptError { + pub fn new(kind: FailureKind, status: Option) -> Self { + Self { + kind, + status, + retry_after: None, + num_retries: None, + failed_deployment_id: None, + detail: None, + classes: Vec::new(), + recorded: false, + names_provider: false, + agentic_followup: false, + outside_deployment: false, + caller_scoped: false, + status_absent: false, + uncounted_request: false, + payload: None, + } + } + + /// The exception's class names, most specific first: the host's, else the kind's. + pub fn class_names(&self) -> Vec<&str> { + if self.classes.is_empty() { + return crate::retry::kind_classes(self.kind).to_vec(); + } + self.classes.iter().map(AsRef::as_ref).collect() + } + + /// `isinstance(error, )`. + pub fn is_a(&self, name: &str) -> bool { + self.class_names().contains(&name) + } + + #[must_use] + pub fn with_retry_after(self, retry_after: Duration) -> Self { + Self { + retry_after: Some(retry_after), + ..self + } + } + + /// A plain Python exception the host raised outside any attempt (a `KeyError` from a stale + /// routing-group lookup, an error a retry policy lookup raised): no status, no provider, + /// nothing recorded against a deployment. `classes` are its class names, most specific first. + pub fn host_raised(classes: &[&str]) -> Self { + Self { + classes: classes.iter().map(|class| (*class).into()).collect(), + recorded: true, + ..Self::new(FailureKind::Other, None) + } + } + + /// The status Python's cooldown rules read: the provider's, else 500. + pub fn cooldown_status(&self) -> u16 { + self.status.unwrap_or(500) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::rate_limit("RateLimitError", FailureKind::RateLimit)] + #[case::context_window("ContextWindowExceededError", FailureKind::ContextWindowExceeded)] + #[case::caller_credential_auth( + "CallerCredentialAuthenticationError", + FailureKind::CallerCredentialAuthentication + )] + #[case::caller_credential_rate_limit( + "CallerCredentialRateLimitError", + FailureKind::CallerCredentialRateLimit + )] + #[case::other("APIError", FailureKind::Other)] + fn failure_kind_round_trips_its_exception_name(#[case] wire: &str, #[case] kind: FailureKind) { + assert_eq!(wire.parse::().unwrap(), kind); + assert_eq!(<&'static str>::from(kind), wire); + } + + #[rstest] + #[case::empty("")] + #[case::lowercase("ratelimiterror")] + fn failure_kind_rejects_unknown_names(#[case] wire: &str) { + assert!(wire.parse::().is_err()); + } +} diff --git a/litellm-rust/crates/router-core/src/fallback.rs b/litellm-rust/crates/router-core/src/fallback.rs new file mode 100644 index 00000000000..a1755bd6ad5 --- /dev/null +++ b/litellm-rust/crates/router-core/src/fallback.rs @@ -0,0 +1,330 @@ +//! Fallback lists as Python spells them, and how a failed group's chain is found in one. +//! Mirrors `get_fallback_model_group`, `get_fallback_model_group_for_lookup_groups`, +//! `check_non_standard_fallback_format` and `fallback_attempt_key` in +//! `router_utils/fallback_event_handlers.py`. +//! +//! A list holds keyed entries (`{"gpt-4": ["gpt-3.5", {"model": "x", "temperature": 0}]}`), +//! bare names (`"claude"`, a chain for every group) and, in a client-side list, dict targets +//! (`{"model": "x", "messages": [...]}`), which override request params for their attempt. + +use serde::Deserialize; +use serde_json::{Map, Value}; + +use std::collections::BTreeMap; + +/// One item of a fallback list, kept as Python holds it. +#[derive(Clone, Debug, PartialEq)] +enum Entry { + Name(String), + Dict(Map), + /// Anything else: Python skips it when it looks a chain up. + Other, +} + +/// A fallback list, in order. `"*"` keys the default chain. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(from = "Vec")] +pub struct Fallbacks(Vec); + +impl From> for Fallbacks { + fn from(entries: Vec) -> Self { + Self( + entries + .into_iter() + .map(|entry| match entry { + Value::String(name) => Entry::Name(name), + Value::Object(dict) => Entry::Dict(dict), + _ => Entry::Other, + }) + .collect(), + ) + } +} + +/// One target of a chain: a group name, or a dict target that also overrides request params. +#[derive(Clone, Debug, PartialEq)] +pub enum FallbackTarget { + Name(String), + Params(Map), +} + +impl FallbackTarget { + fn of(value: &Value) -> Option { + match value { + Value::String(name) => Some(Self::Name(name.clone())), + Value::Object(dict) => Some(Self::Params(dict.clone())), + _ => None, + } + } + + /// `_get_fallback_target_model_group`: the group the target names, if any. + pub fn model(&self) -> Option<&str> { + match self { + Self::Name(name) => Some(name), + Self::Params(dict) => dict.get("model").and_then(Value::as_str), + } + } + + /// The `model` of a dict target that sets a non-string one: the hop routes that value, which + /// no group answers (`_get_fallback_target_model_group` finds no group to check, but + /// `kwargs.update(target)` still rebinds the request's model). + pub fn unroutable_model(&self) -> Option<&Value> { + match self { + Self::Params(dict) => dict.get("model").filter(|model| !model.is_string()), + Self::Name(_) => None, + } + } + + /// `fallback_attempt_key`: a bare name and `{"model": name}` are the same attempt; a dict + /// carrying anything else is its own. + pub fn identity(&self) -> FallbackTarget { + match self { + Self::Params(dict) if dict.len() == 1 => match dict.get("model") { + Some(Value::String(name)) => Self::Name(name.clone()), + _ => self.clone(), + }, + _ => self.clone(), + } + } + + /// The params a dict target merges into the request (`kwargs.update(target)`). + pub fn params(&self) -> Option<&Map> { + match self { + Self::Params(dict) => Some(dict), + Self::Name(_) => None, + } + } +} + +/// What fallback keys match besides the exact group name: Python's `provider_list`, for +/// `"openai/gpt-4o"` to use a `"gpt-4o"` entry, the provider-prefixed spelling of each bare +/// group name (`_provider_prefixed_model_group`), for `"gpt-4o"` to use an `"openai/gpt-4o"` +/// entry, and the `LiteLLMParamsTypedDict` keys that mark a client-side dict target. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default)] +pub struct FallbackKeys { + pub providers: Vec, + pub provider_prefixed: BTreeMap, + pub param_keys: Vec, +} + +impl FallbackKeys { + /// `_check_stripped_model_group`. + fn is_stripped(&self, group: &str, key: &str) -> bool { + self.providers.iter().any(|provider| { + let prefix = format!("{provider}/"); + group.starts_with(&prefix) && group.replace(&prefix, "") == key + }) + } +} + +/// A chain found for a group, and whether it is the `"*"` chain itself. +struct Found { + targets: Vec, + generic: bool, +} + +/// The targets of a keyed entry's value: a list of names and dicts. A bare string is walked +/// character by character, as Python iterates it. +fn chain_of(value: &Value) -> Vec { + match value { + Value::Array(items) => items.iter().filter_map(FallbackTarget::of).collect(), + Value::String(text) => text + .chars() + .map(|c| FallbackTarget::Name(c.to_string())) + .collect(), + _ => Vec::new(), + } +} + +impl Fallbacks { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// `check_non_standard_fallback_format`: a client-side list of names, or of dicts that set + /// a litellm param, is one chain for every group. + pub fn is_client_side(&self, keys: &FallbackKeys) -> bool { + if self.0.is_empty() { + return false; + } + if self.0.iter().all(|entry| matches!(entry, Entry::Name(_))) { + return true; + } + let dicts: Vec<_> = self + .0 + .iter() + .filter_map(|entry| match entry { + Entry::Dict(dict) => Some(dict), + _ => None, + }) + .collect(); + dicts.len() == self.0.len() + && dicts.iter().any(|dict| { + keys.param_keys + .iter() + .any(|key| dict.get(key).is_some_and(|value| !value.is_array())) + }) + } + + /// The whole list as one chain (a client-side list). + pub fn as_chain(&self) -> Vec { + self.0 + .iter() + .filter_map(|entry| match entry { + Entry::Name(name) => Some(FallbackTarget::Name(name.clone())), + Entry::Dict(dict) => Some(FallbackTarget::Params(dict.clone())), + Entry::Other => None, + }) + .collect() + } + + /// `_get_first_default_fallback`: the first target of the first `"*"` entry. + pub fn default_target(&self) -> Option<&str> { + self.0.iter().find_map(|entry| match entry { + Entry::Dict(dict) => match dict.get("*") { + Some(Value::Array(targets)) => targets.first().and_then(Value::as_str), + _ => None, + }, + _ => None, + }) + } + + /// `_has_default_fallbacks`: some entry keys `"*"`. + pub fn has_star(&self) -> bool { + self.0 + .iter() + .any(|entry| matches!(entry, Entry::Dict(dict) if dict.contains_key("*"))) + } + + /// `get_fallback_model_group`. + fn lookup(&self, group: &str, keys: &FallbackKeys) -> Option { + let prefixed = keys.provider_prefixed.get(group).map(String::as_str); + let mut generic = None; + let mut stripped = None; + let mut found = None; + for entry in &self.0 { + match entry { + Entry::Dict(dict) => { + let Some((key, value)) = dict.iter().next() else { + continue; + }; + if key == group { + found = Some(chain_of(value)); + break; + } else if Some(key.as_str()) == prefixed || keys.is_stripped(group, key) { + stripped = Some(chain_of(value)); + } else if key == "*" { + generic = Some(value); + } + } + Entry::Name(name) => found = Some(vec![FallbackTarget::Name(name.clone())]), + Entry::Other => {} + } + } + found + .or(stripped) + .map(|targets| Found { + targets, + generic: false, + }) + .or_else(|| { + generic.map(|value| Found { + targets: chain_of(value), + generic: true, + }) + }) + } + + /// `get_fallback_model_group_for_lookup_groups`: the first lookup group with its own chain + /// wins; `"*"` applies only after every group missed. + pub fn lookup_groups( + &self, + groups: &[&str], + keys: &FallbackKeys, + ) -> Option> { + let results: Vec = groups + .iter() + .filter_map(|group| self.lookup(group, keys)) + .collect(); + let index = results + .iter() + .position(|found| !found.generic) + .or_else(|| (!results.is_empty()).then_some(0))?; + results.into_iter().nth(index).map(|found| found.targets) + } + + /// `_has_fallback_group`'s view: only keyed entries whose targets are names or dicts count. + pub fn has_keyed_chain(&self, group: &str, keys: &FallbackKeys) -> bool { + let keyed = Fallbacks( + self.0 + .iter() + .filter(|entry| matches!(entry, Entry::Dict(_))) + .cloned() + .collect(), + ); + keyed + .lookup(group, keys) + .is_some_and(|found| !found.targets.is_empty()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::an_int(json!({"model": 5}), true)] + #[case::none(json!({"model": null}), true)] + #[case::a_string(json!({"model": "b"}), false)] + #[case::no_model(json!({"temperature": 0}), false)] + fn only_a_set_non_string_model_is_unroutable(#[case] target: Value, #[case] unroutable: bool) { + let target = FallbackTarget::of(&target).unwrap(); + assert_eq!(target.unroutable_model().is_some(), unroutable); + } + + fn names(targets: Option>) -> Option> { + targets.map(|targets| { + targets + .iter() + .map(|t| t.model().unwrap_or_default().to_owned()) + .collect() + }) + } + + #[rstest] + #[case::own_entry(json!([{"a": ["b"]}, {"*": ["d"]}]), "a", Some(vec!["b"]))] + #[case::star_entry(json!([{"a": ["b"]}, {"*": ["d"]}]), "x", Some(vec!["d"]))] + #[case::string_is_every_groups_chain(json!(["b", {"x": ["c"]}]), "a", Some(vec!["b"]))] + #[case::keyed_entry_wins(json!(["c", {"a": ["b"]}]), "a", Some(vec!["b"]))] + #[case::string_value_is_iterated(json!([{"a": "bc"}]), "a", Some(vec!["b", "c"]))] + #[case::none(json!([{"x": ["c"]}]), "a", None)] + fn chains_follow_python( + #[case] list: Value, + #[case] group: &str, + #[case] expected: Option>, + ) { + let fallbacks: Fallbacks = serde_json::from_value(list).unwrap(); + assert_eq!( + names(fallbacks.lookup_groups(&[group], &FallbackKeys::default())), + expected.map(|e| e.into_iter().map(str::to_owned).collect()) + ); + } + + #[rstest] + #[case::names(json!(["a", "b"]), true)] + #[case::param_dicts(json!([{"model": "a", "temperature": 1}]), true)] + #[case::keyed(json!([{"a": ["b"]}]), false)] + #[case::mixed(json!(["a", {"a": ["b"]}]), false)] + fn client_side_lists_follow_python(#[case] list: Value, #[case] expected: bool) { + let fallbacks: Fallbacks = serde_json::from_value(list).unwrap(); + let keys = FallbackKeys { + param_keys: vec!["model".to_owned(), "temperature".to_owned()], + ..FallbackKeys::default() + }; + assert_eq!(fallbacks.is_client_side(&keys), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/group_info.rs b/litellm-rust/crates/router-core/src/group_info.rs new file mode 100644 index 00000000000..9a49858ca18 --- /dev/null +++ b/litellm-rust/crates/router-core/src/group_info.rs @@ -0,0 +1,646 @@ +//! Model group info and the configured-value reads: what the router reports a model group, or a +//! concrete deployment behind a listed name, can do. +//! +//! The rules are the Python router core's (`_set_model_group_info`, `get_model_listing_info`, +//! `get_configured_token_limits`, `get_configured_mode`, `get_configured_display_name`, +//! `get_configured_service_tiers`). The host picks the deployments (the registry lookups) and +//! answers what needs its objects as [`Host::model_info`] lookups asked while the query runs: a +//! deployment's validated params, its merged model info and provider, its capabilities, and the +//! group's first entry as its model validates it. The core decides which deployments count and +//! how their values combine. + +use serde_json::{Map, Number, Value}; + +use crate::{ + error::Error, + host::{Host, ModelInfoLookup}, + model_info::{ + discovered, is_opaque, lookup, malformed, python_float, python_int, raised, truthy, + without_digit_underscores, + }, +}; + +/// One `_set_model_group_info` query. +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(default)] +pub struct GroupInfoQuery { + /// The model group the deployments serve (an alias already resolved). + pub model_group: String, + /// The name the group is reported as. + pub user_facing_model_group_name: String, + /// The group is a team's public model name the team's deployments serve. + pub serves_team_deployments: bool, + /// A wildcard pattern routes `model_group` (asked only when an entry needs it). + pub pattern_routes: bool, + /// The first string `model_info.description` among the deployments. + pub description: Value, + /// The deployments the group routes to, as stored. + pub deployments: Vec, +} + +/// The group info: `None` when no deployment matched, else the fields the merge assigned after +/// the host built the group from its first deployment ([`ModelInfoLookup::GroupSeed`]). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct GroupInfo { + pub assignments: Map, +} + +/// The type name Python reports for a value, in its error messages. +pub(crate) fn type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) if is_opaque(value) => "object", + Value::Object(_) => "dict", + } +} + +/// A number as Python's arithmetic reads it (a bool is 0 or 1), or `None`. +fn arithmetic(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(Number::from(u8::from(*flag))), + Value::Number(number) => Some(number.clone()), + _ => None, + } +} + +/// `total += value`, with `total` starting at `0`; adding a non-number raises `TypeError`. +fn add(total: Option, value: &Value) -> Result { + let total = total.unwrap_or_else(|| Value::Number(Number::from(0))); + let (Some(left), Some(right)) = (arithmetic(&total), arithmetic(value)) else { + return Err(raised( + "TypeError", + format!( + "unsupported operand type(s) for +=: '{}' and '{}'", + type_name(&total), + type_name(value) + ), + )); + }; + if let (Some(left), Some(right)) = (left.as_i64(), right.as_i64()) + && let Some(sum) = left.checked_add(right) + { + return Ok(Value::Number(Number::from(sum))); + } + let sum = left.as_f64().unwrap_or(0.0) + right.as_f64().unwrap_or(0.0); + Ok(Number::from_f64(sum).map_or(Value::Null, Value::Number)) +} + +/// `left > right`, for numbers or strings; any other pair raises `TypeError`. +fn greater(left: &Value, right: &Value) -> Result { + if let (Some(left), Some(right)) = (arithmetic(left), arithmetic(right)) { + return Ok(left.as_f64().unwrap_or(0.0) > right.as_f64().unwrap_or(0.0)); + } + if let (Value::String(left), Value::String(right)) = (left, right) { + return Ok(left > right); + } + Err(raised( + "TypeError", + format!( + "'>' not supported between instances of '{}' and '{}'", + type_name(left), + type_name(right) + ), + )) +} + +/// `_cost_value_as_float`: `float(value)`, or `None` when it is unset or not a number. +fn cost_as_float(value: Option<&Value>) -> Option { + match value? { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + Value::String(text) => { + without_digit_underscores(text.trim()).and_then(|digits| python_float(&digits)) + } + _ => None, + } +} + +fn float(value: Option) -> Value { + value + .and_then(Number::from_f64) + .map_or(Value::Null, Value::Number) +} + +/// `value.get(key)` on a dict; a value that is no dict raises `AttributeError`, as `.get` does. +fn get<'a>(value: &'a Value, key: &str) -> Result, Error> { + match value { + Value::Object(map) if !is_opaque(value) => Ok(map.get(key)), + other => Err(raised( + "AttributeError", + format!("'{}' object has no attribute 'get'", type_name(other)), + )), + } +} + +/// `model.get(key)`, then `litellm_params.get(key)`, then `model_info.get(key)`: the first that is +/// not `None`. +fn first_set( + model: &Value, + params: &Value, + info: &Value, + key: &str, +) -> Result, Error> { + for layer in [model, params, info] { + if let Some(value) = get(layer, key)?.filter(|value| !value.is_null()) { + return Ok(Some(value.clone())); + } + } + Ok(None) +} + +fn object(answer: Value, kind: &str) -> Result, Error> { + match answer { + Value::Object(map) => Ok(map), + _ => Err(malformed(kind)), + } +} + +/// The group being merged: the host's validated first entry, and what the merge assigned since. +struct Merged { + current: Map, + assignments: Map, +} + +impl Merged { + fn get(&self, field: &str) -> &Value { + self.current.get(field).unwrap_or(&Value::Null) + } + + fn set(&mut self, field: &str, value: Value) { + self.current.insert(field.to_owned(), value.clone()); + self.assignments.insert(field.to_owned(), value); + } + + /// `if info.get(field) is not None and info[field] > group.field (or group.field is None)`. + fn widen(&mut self, field: &str, info: &Map) -> Result<(), Error> { + let Some(value) = info.get(field).filter(|value| !value.is_null()) else { + return Ok(()); + }; + let current = self.get(field); + if current.is_null() || greater(value, current)? { + self.set(field, value.clone()); + } + Ok(()) + } + + /// The cost `field` widened to `float(info[field])` when that exceeds `group.field or 0.0`. + fn widen_cost(&mut self, field: &str, info: &Map) { + let Some(cost) = cost_as_float(info.get(field)) else { + return; + }; + let current = self.get(field); + let floor = if truthy(current) { + current.as_f64().unwrap_or(0.0) + } else { + 0.0 + }; + if current.is_null() || cost > floor { + self.set(field, float(Some(cost))); + } + } + + /// `if info.get(field) is True: group.field = True`. + fn any(&mut self, field: &str, info: &Map) { + if info.get(field) == Some(&Value::Bool(true)) { + self.set(field, Value::Bool(true)); + } + } +} + +/// `tuple(dict.fromkeys((*current, *added))) or None`. +fn union(current: &Value, added: &[Value]) -> Value { + let mut kept: Vec = Vec::new(); + let existing = current.as_array().map(Vec::as_slice).unwrap_or_default(); + for value in existing.iter().chain(added) { + if !kept.contains(value) { + kept.push(value.clone()); + } + } + if kept.is_empty() { + Value::Null + } else { + Value::Array(kept) + } +} + +/// `_set_model_group_info`: the group's info merged across the deployments that serve it. +pub async fn model_group_info( + host: &dyn Host, + query: &GroupInfoQuery, +) -> Result, Error> { + let mut merged: Option = None; + let mut totals: [(&str, Option); 4] = + [("tpm", None), ("rpm", None), ("itpm", None), ("otpm", None)]; + let mut clientside_auth_params = Value::Null; + let mut efforts_initialized = false; + let mut efforts_unknown = false; + let empty = Value::Object(Map::new()); + + for (index, model) in query.deployments.iter().enumerate() { + let named = get(model, "model_name")?; + let is_match = query.serves_team_deployments + || named.is_some_and(|name| name.as_str() == Some(query.model_group.as_str())) + || (named.is_some() && query.pattern_routes); + if !is_match { + continue; + } + let params = object( + lookup(host, ModelInfoLookup::GroupParams { index }).await?, + "group_params", + )?; + clientside_auth_params = params + .get("configurable_clientside_auth_params") + .cloned() + .unwrap_or(Value::Null); + let model_params = get(model, "litellm_params")?.unwrap_or(&empty); + let model_info_dict = get(model, "model_info")?.unwrap_or(&empty); + let mut deployment = [None, None, None, None]; + for (slot, (metric, _)) in deployment.iter_mut().zip(totals.iter()) { + *slot = first_set(model, model_params, model_info_dict, metric)?; + } + + let facts = object( + lookup( + host, + ModelInfoLookup::GroupDeployment { + index, + model: params.get("model").cloned().unwrap_or(Value::Null), + }, + ) + .await?, + "group_deployment", + )?; + let mapped = facts.get("mapped") == Some(&Value::Bool(true)); + let litellm_model = facts.get("litellm_model").cloned().unwrap_or(Value::Null); + let llm_provider = facts.get("llm_provider").cloned().unwrap_or(Value::Null); + let model_info = match facts.get("model_info") { + Some(Value::Object(info)) if !is_opaque(&Value::Object(info.clone())) => info.clone(), + Some(Value::Null) | None => { + // No model info: what the deployment's own (database) model_info says. + let db_info = get(model, "model_info")?.unwrap_or(&empty); + let mut info = Map::new(); + info.insert("key".to_owned(), Value::String(query.model_group.clone())); + for key in ["max_tokens", "max_input_tokens", "max_output_tokens"] { + info.insert(key.to_owned(), Value::Null); + } + for key in ["input_cost_per_token", "output_cost_per_token"] { + info.insert(key.to_owned(), float(cost_as_float(get(db_info, key)?))); + } + info.insert("litellm_provider".to_owned(), llm_provider.clone()); + info.insert( + "mode".to_owned(), + match get(db_info, "mode")? { + Some(mode) => mode.clone(), + None => Value::String("chat".to_owned()), + }, + ); + info.insert( + "supported_openai_params".to_owned(), + match facts.get("supported_openai_params") { + Some(Value::Null) | None => Value::Array(Vec::new()), + Some(params) => params.clone(), + }, + ); + info.insert("supports_system_messages".to_owned(), Value::Null); + info + } + Some(_) => return Err(malformed("group_deployment")), + }; + + match merged.as_mut() { + None => { + let mut seed = Map::new(); + seed.insert( + "model_group".to_owned(), + Value::String(query.user_facing_model_group_name.clone()), + ); + seed.insert( + "providers".to_owned(), + Value::Array(vec![llm_provider.clone()]), + ); + seed.insert("description".to_owned(), query.description.clone()); + seed.extend(model_info.clone()); + seed.insert( + "supports_continuation".to_owned(), + Value::Bool( + model_info.get("supports_continuation") == Some(&Value::Bool(true)), + ), + ); + seed.insert("supports_fast_mode".to_owned(), Value::Bool(true)); + seed.insert("supported_reasoning_efforts".to_owned(), Value::Null); + seed.insert("supported_endpoints".to_owned(), Value::Null); + let current = object( + lookup(host, ModelInfoLookup::GroupSeed { info: seed }).await?, + "group_seed", + )?; + merged = Some(Merged { + current, + assignments: Map::new(), + }); + } + Some(group) => { + let providers = group + .get("providers") + .as_array() + .cloned() + .unwrap_or_default(); + if !providers.contains(&llm_provider) { + let mut providers = providers; + providers.push(llm_provider.clone()); + group.set("providers", Value::Array(providers)); + } + group.widen("max_input_tokens", &model_info)?; + group.widen("max_output_tokens", &model_info)?; + group.widen_cost("input_cost_per_token", &model_info); + group.widen_cost("output_cost_per_token", &model_info); + for field in [ + "supports_parallel_function_calling", + "supports_vision", + "supports_function_calling", + "supports_web_search", + "supports_url_context", + "supports_continuation", + "supports_reasoning", + ] { + group.any(field, &model_info); + } + if let Some(params) = model_info + .get("supported_openai_params") + .filter(|params| !params.is_null()) + { + group.set("supported_openai_params", params.clone()); + } + for (slot, metric) in deployment.iter_mut().zip(["tpm", "rpm"]) { + if slot.is_none() + && let Some(value) = model_info.get(metric).filter(|v| !v.is_null()) + { + *slot = Some(value.clone()); + } + } + } + } + let Some(group) = merged.as_mut() else { + unreachable!("the group was just built"); + }; + + let capabilities = object( + lookup( + host, + ModelInfoLookup::GroupCapabilities { + index, + model_info: Value::Object(model_info), + litellm_model, + llm_provider, + mapped, + }, + ) + .await?, + "group_capabilities", + )?; + let endpoints = capabilities + .get("supported_endpoints") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let endpoints = union(group.get("supported_endpoints"), &endpoints); + group.set("supported_endpoints", endpoints); + let fast_mode = truthy(group.get("supports_fast_mode")) + && capabilities.get("supports_fast_mode") == Some(&Value::Bool(true)); + group.set("supports_fast_mode", Value::Bool(fast_mode)); + match capabilities + .get("reasoning_efforts") + .cloned() + .unwrap_or(Value::Null) + { + Value::Null => { + efforts_unknown = true; + group.set("supported_reasoning_efforts", Value::Null); + } + resolved if !efforts_initialized => { + efforts_initialized = true; + if !efforts_unknown { + group.set("supported_reasoning_efforts", resolved); + } + } + resolved if !efforts_unknown => { + let intersected = lookup( + host, + ModelInfoLookup::ReasoningIntersect { + current: group.get("supported_reasoning_efforts").clone(), + resolved, + }, + ) + .await?; + group.set("supported_reasoning_efforts", intersected); + } + _ => {} + } + + for ((_, total), value) in totals.iter_mut().zip(deployment) { + if let Some(value) = value { + *total = Some(add(total.take(), &value)?); + } + } + } + + let Some(mut group) = merged else { + return Ok(None); + }; + for (metric, total) in totals { + if let Some(total) = total { + group.set(metric, total); + } + } + if !clientside_auth_params.is_null() { + group.set( + "configurable_clientside_auth_params", + clientside_auth_params, + ); + } + Ok(Some(GroupInfo { + assignments: group.assignments, + })) +} + +/// `coerce_token_limit`: a token limit as an int, `None` when it is unset or not a usable number +/// (a bool never is). +pub fn coerce_token_limit(value: Option<&Value>) -> Option { + match value? { + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .filter(|n| n.is_finite()) + .map(|n| n.trunc() as i64) + }), + Value::String(text) => without_digit_underscores(text.trim()) + .and_then(|digits| python_int(&digits)) + .and_then(|number| number.as_i64()), + _ => None, + } +} + +/// The deployment's discovered limits under its configured `model_info` values that are set. +async fn effective_model_info( + host: &dyn Host, + id: Value, + configured: &Map, +) -> Result, Error> { + let mut info = discovered(host, id).await?; + info.extend( + configured + .iter() + .filter(|(_, value)| !value.is_null()) + .map(|(key, value)| (key.clone(), value.clone())), + ); + Ok(info) +} + +/// `get_configured_token_limits`: the (max input, max output) tokens a concrete deployment +/// configures, or its refresh discovered, from its `model_info` (`exclude_none` dump) and id. +pub async fn configured_token_limits( + host: &dyn Host, + id: Value, + configured: &Map, +) -> Result<(Option, Option), Error> { + let info = effective_model_info(host, id, configured).await?; + Ok(( + coerce_token_limit(info.get("max_input_tokens")), + coerce_token_limit(info.get("max_output_tokens")), + )) +} + +/// `get_configured_mode` / `get_configured_display_name`: a configured string that is not blank. +pub fn configured_text(value: Option<&Value>) -> Option { + value + .and_then(Value::as_str) + .filter(|text| !text.trim().is_empty()) + .map(str::to_owned) +} + +/// `get_configured_service_tiers`: each deployment's `model_info.service_tiers`, unvalidated. +pub fn configured_service_tiers(model_infos: &[Map]) -> Vec { + model_infos + .iter() + .map(|info| info.get("service_tiers").cloned().unwrap_or(Value::Null)) + .collect() +} + +/// What a listed name's deployments contribute to its `/v1/models` entry. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ListingInfo { + pub cost_map_keys: Vec, + pub max_input_tokens: Option, + pub max_output_tokens: Option, +} + +fn named_text(value: Option<&Value>) -> Option<&str> { + value + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) +} + +/// `deployment_pricing_models(deployment)[0]`: `model_info.base_model`, then +/// `litellm_params.base_model`, then `litellm_params.model`; malformed blocks count as unset. +fn pricing_key(deployment: &Value) -> Option { + let block = |key: &str| { + deployment + .get(key) + .and_then(Value::as_object) + .filter(|_| !is_opaque(deployment.get(key).unwrap_or(&Value::Null))) + }; + let params = block("litellm_params"); + let info = block("model_info"); + let base_model = info + .and_then(|info| info.get("base_model")) + .filter(|base| truthy(base)) + .or_else(|| params.and_then(|params| params.get("base_model"))); + named_text(base_model) + .or_else(|| named_text(params.and_then(|params| params.get("model")))) + .map(str::to_owned) +} + +/// `get_model_listing_info` over the deployments a listed name names: the cost-map keys of their +/// models, deduplicated, and the widest token limits they configure or discovered. +pub async fn model_listing_info( + host: &dyn Host, + deployments: &[Value], +) -> Result { + let mut listing = ListingInfo::default(); + let empty = Value::Object(Map::new()); + for deployment in deployments { + let info = match get(deployment, "model_info")? { + Some(info) if truthy(info) => info, + _ => &empty, + }; + let id = get(info, "id")?.cloned().unwrap_or(Value::Null); + let configured = match info { + Value::Object(map) => map.clone(), + _ => Map::new(), + }; + let effective = effective_model_info(host, id, &configured).await?; + for (field, widest) in [ + ("max_input_tokens", &mut listing.max_input_tokens), + ("max_output_tokens", &mut listing.max_output_tokens), + ] { + if let Some(limit) = coerce_token_limit(effective.get(field)) { + *widest = Some(widest.map_or(limit, |current| current.max(limit))); + } + } + } + for key in deployments.iter().filter_map(pricing_key) { + if !listing.cost_map_keys.contains(&key) { + listing.cost_map_keys.push(key); + } + } + Ok(listing) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn coerce_token_limit_reads_python_ints() { + assert_eq!(coerce_token_limit(Some(&json!(128))), Some(128)); + assert_eq!(coerce_token_limit(Some(&json!(12.9))), Some(12)); + assert_eq!(coerce_token_limit(Some(&json!(" 1_000 "))), Some(1000)); + assert_eq!(coerce_token_limit(Some(&json!("128,000"))), None); + assert_eq!(coerce_token_limit(Some(&json!(true))), None); + assert_eq!(coerce_token_limit(None), None); + } + + #[test] + fn totals_add_like_python() { + assert_eq!(add(None, &json!(5)).unwrap(), json!(5)); + assert_eq!(add(Some(json!(5)), &json!(true)).unwrap(), json!(6)); + assert_eq!(add(Some(json!(5)), &json!(0.5)).unwrap(), json!(5.5)); + assert!(add(Some(json!(5)), &json!("1")).is_err()); + } + + #[test] + fn union_keeps_first_seen_order_and_empty_reads_as_none() { + assert_eq!( + union(&json!(["a", "b"]), &[json!("b"), json!("c")]), + json!(["a", "b", "c"]) + ); + assert_eq!(union(&Value::Null, &[]), Value::Null); + } + + #[test] + fn pricing_key_prefers_the_base_model() { + let deployment = json!({ + "litellm_params": {"model": "azure/x", "base_model": "params-base"}, + "model_info": {"base_model": "info-base"}, + }); + assert_eq!(pricing_key(&deployment).as_deref(), Some("info-base")); + let deployment = json!({"litellm_params": {"model": "azure/x"}, "model_info": "junk"}); + assert_eq!(pricing_key(&deployment).as_deref(), Some("azure/x")); + } +} diff --git a/litellm-rust/crates/router-core/src/health.rs b/litellm-rust/crates/router-core/src/health.rs new file mode 100644 index 00000000000..020a1a0f47d --- /dev/null +++ b/litellm-rust/crates/router-core/src/health.rs @@ -0,0 +1,960 @@ +//! [`Health`]: cooldowns, allowed-fails counters and per-minute usage, kept in the shared +//! [`RouterCache`] under the Python router's keys, encodings and TTLs: +//! +//! | state | key | value | TTL | +//! |---|---|---|---| +//! | cooldown | `deployment:{id}:cooldown` | `{exception_received, status_code, timestamp, cooldown_time}` | cooldown time | +//! | allowed fails | `deployment:{id}:allowed_fails[:{suffix}]` | int counter | cooldown time | +//! | usage | `global_router:{id}:{model}:tpm:{HH-MM}` and `:rpm:` | int counter | 60s | +//! | outcomes (local only) | `{id}:successes`, `{id}:fails` | int counter | 60s | +//! +//! From `router_utils/cooldown_cache.py`, `router_utils/cooldown_handlers.py`, +//! `router_utils/router_callbacks/track_deployment_metrics.py`, `RouterCacheEnum` in +//! `types/router.py` and `Router._increment_usage` / `deployment_callback_on_failure` in +//! `router.py`. + +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use serde::{Deserialize, Serialize}; +use tracing::debug; + +use crate::{ + cache::RouterCache, + error::{AttemptError, Error, FailureKind}, + outcome::{OutcomeCallbacks, OutcomeWrite}, + registry::{Deployment, Registry}, + settings::Settings, +}; + +/// `RoutingArgs.ttl`: per-minute usage keys expire after a minute. +pub const USAGE_TTL: Duration = Duration::from_secs(60); +/// `DEFAULT_FAILURE_THRESHOLD_PERCENT`. +pub const FAILURE_THRESHOLD_PERCENT: f64 = 0.5; +/// `DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS`. +pub const FAILURE_THRESHOLD_MINIMUM_REQUESTS: u64 = 5; +/// `SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD`. +pub const SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: u64 = 1000; +/// `exception_masker` in `CooldownCache`: the visible prefix of `exception_received`. +const EXCEPTION_VISIBLE_PREFIX: usize = 50; + +pub trait Clock: Send + Sync { + fn now(&self) -> SystemTime; +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> SystemTime { + SystemTime::now() + } +} + +fn unix_seconds(time: SystemTime) -> f64 { + time.duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs_f64()) + .unwrap_or_default() +} + +/// `get_utc_datetime().strftime("%H-%M")`. +pub fn current_minute(time: SystemTime) -> String { + let seconds = time + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs()) + .unwrap_or_default() + % 86_400; + format!("{:02}-{:02}", seconds / 3600, (seconds % 3600) / 60) +} + +pub fn cooldown_key(id: &str) -> String { + format!("deployment:{id}:cooldown") +} + +pub fn allowed_fails_key(id: &str, suffix: Option<&str>) -> String { + match suffix { + Some(suffix) => format!("deployment:{id}:allowed_fails:{suffix}"), + None => format!("deployment:{id}:allowed_fails"), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::EnumString)] +pub enum UsageMetric { + #[strum(serialize = "tpm")] + Tpm, + #[strum(serialize = "rpm")] + Rpm, + /// Input tokens per minute, which the io-token rate limit check counts. + #[strum(serialize = "itpm")] + Itpm, + /// Output tokens per minute. + #[strum(serialize = "otpm")] + Otpm, +} + +/// `RouterCacheEnum.TPM` / `RPM` / `ITPM` / `OTPM`: `model` is the deployment's +/// `litellm_params.model`. +pub fn usage_key(id: &str, model: &str, metric: UsageMetric, minute: &str) -> String { + format!("global_router:{id}:{model}:{metric}:{minute}") +} + +/// `CooldownCacheValue`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CooldownValue { + pub exception_received: String, + pub status_code: String, + pub timestamp: f64, + /// Whole seconds are written as an int, as Python's default `cooldown_time` is, so the + /// "Try again in N seconds" text reads the same. + #[serde(serialize_with = "whole_as_int")] + pub cooldown_time: f64, +} + +fn whole_as_int(value: &f64, serializer: S) -> Result { + if value.fract() == 0.0 && value.abs() < i64::MAX as f64 { + serializer.serialize_i64(*value as i64) + } else { + serializer.serialize_f64(*value) + } +} + +/// `litellm.should_retry`. +pub fn should_retry_status(status: u16) -> bool { + matches!(status, 408 | 409 | 429) || status >= 500 +} + +/// `_is_cooldown_required`: a 4xx cools down only for 429, 401, 402, 404 and 408, every other +/// status does, and a connection error or an empty status never does. +pub fn is_cooldown_required(failure: &AttemptError) -> bool { + if failure.kind == FailureKind::ApiConnection || failure.status_absent { + return false; + } + match failure.cooldown_status() { + status @ 400..500 => matches!(status, 429 | 401 | 402 | 408 | 404), + _ => true, + } +} + +/// `SensitiveDataMasker(visible_prefix=50, visible_suffix=0, mask_short_values=False)`. +fn mask_exception(text: &str) -> String { + let length = text.chars().count(); + if length <= EXCEPTION_VISIBLE_PREFIX { + return text.to_owned(); + } + let mut masked: String = text.chars().take(EXCEPTION_VISIBLE_PREFIX).collect(); + masked.extend(std::iter::repeat_n('*', length - EXCEPTION_VISIBLE_PREFIX)); + masked +} + +/// `_EXCEPTION_POLICY_FIELDS`: a deployment policy's fields in the order Python matches them +/// (a content-policy violation is a bad request, so it comes first). +const DEPLOYMENT_POLICY_FIELDS: [(&str, &str); 9] = [ + ( + "ContentPolicyViolationError", + "ContentPolicyViolationErrorAllowedFails", + ), + ("BadRequestError", "BadRequestErrorAllowedFails"), + ("AuthenticationError", "AuthenticationErrorAllowedFails"), + ("Timeout", "TimeoutErrorAllowedFails"), + ("RateLimitError", "RateLimitErrorAllowedFails"), + ("InternalServerError", "InternalServerErrorAllowedFails"), + ( + "ServiceUnavailableError", + "ServiceUnavailableErrorAllowedFails", + ), + ("BadGatewayError", "BadGatewayErrorAllowedFails"), + ("NotFoundError", "NotFoundErrorAllowedFails"), +]; + +/// `_resolve_allowed_fails_from_policy` over a deployment's `model_info.allowed_fails_policy`: +/// the first field set for a class the failure is an instance of. +pub fn deployment_allowed_fails( + policy: Option<&serde_json::Map>, + failure: &AttemptError, +) -> Option { + let policy = policy?; + DEPLOYMENT_POLICY_FIELDS + .iter() + .filter(|(class, _)| failure.is_a(class)) + .find_map(|(_, field)| policy.get(*field).and_then(serde_json::Value::as_f64)) +} + +/// What the cooldown rules need to know about the deployment's group. +#[derive(Clone, Copy, Debug, Default)] +pub struct GroupShape { + /// The deployment is the only one in its model group. + pub single_deployment: bool, + /// The requested group has a fallback configured. + pub has_fallback_group: bool, +} + +/// `_core_team_model_has_alternatives`: another routable deployment serves the deployment's team +/// public name. +pub(crate) fn team_model_has_alternatives(registry: &Registry, deployment: &Deployment) -> bool { + let info = &deployment.model_info; + let (Some(team_id), Some(public_name)) = (&info.team_id, &info.team_public_model_name) else { + return false; + }; + registry + .team_named(team_id, public_name) + .is_some_and(|siblings| { + siblings + .iter() + .filter(|sibling| !sibling.model_info.blocked) + .count() + > 1 + }) +} + +/// `_core_routing_group_has_alternatives`: `model_group` names a callable routing group whose +/// members span more than one deployment. +pub(crate) fn routing_group_has_alternatives(registry: &Registry, model_group: &str) -> bool { + let resolved = registry.alias_of(model_group).unwrap_or(model_group); + registry.routing_group_named(resolved).is_some_and(|group| { + group + .models + .iter() + .map(|member| registry.named(member).map_or(0, |members| members.len())) + .sum::() + > 1 + }) +} + +/// An outcome the router did not see itself (`Health.report`). +#[derive(Clone, Debug)] +pub enum Outcome { + Success { total_tokens: u64 }, + Failure(AttemptError), +} + +#[derive(Clone)] +pub struct Health { + cache: Arc, + clock: Arc, +} + +impl Health { + pub fn new(cache: Arc, clock: Arc) -> Self { + Self { cache, clock } + } + + pub fn cache(&self) -> &Arc { + &self.cache + } + + pub(crate) fn now_time(&self) -> SystemTime { + self.clock.now() + } + + fn now(&self) -> f64 { + unix_seconds(self.clock.now()) + } + + /// The ids among `ids` with an unexpired cooldown (`async_get_active_cooldowns`). + pub async fn active_cooldowns( + &self, + ids: &[String], + ) -> Result, Error> { + let keys: Vec = ids.iter().map(|id| cooldown_key(id)).collect(); + let values = self.cache.async_batch_get_cache(&keys).await?; + let now = self.now(); + Ok(ids + .iter() + .zip(values) + .filter_map(|(id, value)| { + let value: CooldownValue = serde_json::from_value(value?).ok()?; + (value.timestamp + value.cooldown_time - now > 0.0).then(|| (id.clone(), value)) + }) + .collect()) + } + + /// `add_deployment_to_cooldown`. + pub async fn add_cooldown( + &self, + id: &str, + failure: &AttemptError, + cooldown: Duration, + ) -> Result<(), Error> { + let value = CooldownValue { + exception_received: mask_exception(failure.detail.as_deref().unwrap_or_default()), + status_code: failure.cooldown_status().to_string(), + timestamp: self.now(), + cooldown_time: cooldown.as_secs_f64(), + }; + let value = serde_json::to_value(value).unwrap_or_default(); + self.cache + .async_set_cache(&cooldown_key(id), value, Some(cooldown)) + .await?; + Ok(()) + } + + async fn local_count(&self, key: &str) -> Result { + Ok(self + .cache + .async_get_local(key) + .await? + .and_then(|value| value.as_f64()) + .map_or(0, |count| count as u64)) + } + + /// The per-minute (successes, fails) this worker saw (`{id}:successes`, `{id}:fails`, kept + /// `local_only`). + pub async fn outcomes_this_minute(&self, id: &str) -> Result<(u64, u64), Error> { + Ok(( + self.local_count(&format!("{id}:successes")).await?, + self.local_count(&format!("{id}:fails")).await?, + )) + } + + /// `increment_deployment_successes_for_current_minute`: returns the key it counted under. + pub async fn count_success(&self, id: &str) -> Result { + let key = format!("{id}:successes"); + self.cache + .async_increment_local(&key, 1.0, Some(USAGE_TTL)) + .await?; + Ok(key) + } + + /// `increment_deployment_failures_for_current_minute`. + pub async fn count_failure(&self, id: &str) -> Result<(), Error> { + self.cache + .async_increment_local(&format!("{id}:fails"), 1.0, Some(USAGE_TTL)) + .await?; + Ok(()) + } + + /// Cooldown duration precedence from `deployment_callback_on_failure`: the deployment's + /// `cooldown_time`, then the provider's retry-after, then the router default. + pub fn cooldown_duration( + settings: &Settings, + deployment: &Deployment, + failure: &AttemptError, + ) -> Duration { + Self::cooldown_for_call(settings, deployment, None, failure) + } + + /// `cooldown_duration` for a call whose own params carry `cooldown_time` (a request's + /// value wins over the deployment's `litellm_params`, not over its `model_info`). + pub fn cooldown_for_call( + settings: &Settings, + deployment: &Deployment, + call_cooldown: Option, + failure: &AttemptError, + ) -> Duration { + let configured = deployment + .model_info + .cooldown_time + .or(call_cooldown) + .or_else(|| { + deployment + .raw_params + .get("cooldown_time") + .and_then(|v| v.as_f64()) + }) + .filter(|seconds| *seconds >= 0.0) + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()); + configured + .or(failure.retry_after) + .unwrap_or(settings.cooldown_time) + } + + /// The deployment's own `cooldown_time`: `model_info` first, then `litellm_params` + /// (`_first_present(model_info, litellm_params, key="cooldown_time")`). It is both the + /// cooldown and the TTL of the deployment's `allowed_fails:generic` counter. + fn configured_cooldown(deployment: &Deployment) -> Option { + deployment + .model_info + .cooldown_time + .or_else(|| { + deployment + .raw_params + .get("cooldown_time") + .and_then(|v| v.as_f64()) + }) + .filter(|seconds| *seconds >= 0.0) + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + } + + /// Record a failed attempt and cool the deployment down when the rules say so. Returns the + /// cooldown applied, if any. + pub async fn record_failure( + &self, + settings: &Settings, + deployment: &Deployment, + shape: GroupShape, + call_cooldown: Option, + failure: &AttemptError, + ) -> Result, Error> { + if failure.outside_deployment { + return Ok(None); + } + self.count_failure(&deployment.id).await?; + let cooldown = Self::cooldown_for_call(settings, deployment, call_cooldown, failure); + self.cool_down_after(settings, deployment, shape, failure, cooldown) + .await + } + + /// `set_cooldown_deployments` for a failure already counted: cool the deployment down for + /// `cooldown` when the rules say so. Returns the cooldown applied, if any. + pub async fn cool_down_after( + &self, + settings: &Settings, + deployment: &Deployment, + shape: GroupShape, + failure: &AttemptError, + cooldown: Duration, + ) -> Result, Error> { + if !Self::should_run_cooldown_logic(settings, deployment, failure, cooldown) { + return Ok(None); + } + if !self + .should_cooldown(settings, deployment, shape, failure) + .await? + { + return Ok(None); + } + debug!(deployment_id = %deployment.id, cooldown_seconds = cooldown.as_secs_f64(), "cooling deployment down"); + self.add_cooldown(&deployment.id, failure, cooldown).await?; + Ok(Some(cooldown)) + } + + /// `_should_run_cooldown_logic`. + fn should_run_cooldown_logic( + settings: &Settings, + deployment: &Deployment, + failure: &AttemptError, + cooldown: Duration, + ) -> bool { + !matches!( + failure.kind, + FailureKind::Declined + | FailureKind::CallerCredentialAuthentication + | FailureKind::CallerCredentialRateLimit + ) && !failure.caller_scoped + && !cooldown.is_zero() + && !settings.disable_cooldowns + && (is_cooldown_required(failure) + || deployment_allowed_fails( + deployment.model_info.allowed_fails_policy.as_ref(), + failure, + ) + .is_some()) + && !settings + .provider_default_deployment_ids + .contains(&deployment.id) + } + + /// `_should_cooldown_deployment`. + async fn should_cooldown( + &self, + settings: &Settings, + deployment: &Deployment, + shape: GroupShape, + failure: &AttemptError, + ) -> Result { + let status = failure.cooldown_status(); + let policy_fails = settings.allowed_fails_for(failure); + let lone = shape.single_deployment && !shape.has_fallback_group; + let deployment_policy = deployment.model_info.allowed_fails_policy.as_ref(); + let deployment_fails = deployment_allowed_fails(deployment_policy, failure); + if shape.single_deployment + && status == 402 + && deployment_fails.is_none() + && policy_fails.is_none() + { + return Ok(false); + } + let deployment_allowed = deployment.model_info.allowed_fails; + if deployment_policy.is_some() || deployment_allowed.is_some() { + // `_should_cooldown_based_on_deployment_policy`: a deployment-wide `allowed_fails` + // keeps the lone-deployment safety net; a policy entry naming the error does not. + if deployment_fails.is_none() + && lone + && (deployment_allowed.is_some() || policy_fails.is_none()) + { + return Ok(false); + } + let (allowed, suffix) = match deployment_fails { + Some(allowed) => ( + Some(allowed), + failure.class_names().first().map(|name| (*name).to_owned()), + ), + None => ( + deployment_allowed.map(f64::from), + deployment_allowed.map(|_| "generic".to_owned()), + ), + }; + let allowed = allowed + .or_else(|| policy_fails.map(f64::from)) + .unwrap_or_else(|| f64::from(settings.effective_allowed_fails())); + let ttl = + Self::configured_cooldown(deployment).unwrap_or_else(|| settings.fails_window()); + return self + .exceeds_allowed_fails(&deployment.id, suffix.as_deref(), allowed, ttl) + .await; + } + if settings.allowed_fails_policy.is_none() && !settings.allowed_fails_set() { + let (successes, fails) = self.outcomes_this_minute(&deployment.id).await?; + let total = successes + fails; + let percent_fails = if total > 0 { + fails as f64 / total as f64 + } else { + 0.0 + }; + return Ok((status == 429 && !shape.single_deployment) + || (percent_fails == 1.0 && total >= SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD) + || (percent_fails > FAILURE_THRESHOLD_PERCENT + && total >= FAILURE_THRESHOLD_MINIMUM_REQUESTS + && !shape.single_deployment) + || !should_retry_status(status)); + } + if lone && policy_fails.is_none() { + return Ok(false); + } + let allowed = policy_fails.unwrap_or_else(|| settings.effective_allowed_fails()); + self.exceeds_allowed_fails( + &deployment.id, + None, + f64::from(allowed), + settings.fails_window(), + ) + .await + } + + /// `should_cooldown_based_on_allowed_fails_policy`: count this failure, then compare. + async fn exceeds_allowed_fails( + &self, + id: &str, + suffix: Option<&str>, + allowed: f64, + ttl: Duration, + ) -> Result { + let fails = self + .cache + .async_increment(&allowed_fails_key(id, suffix), 1.0, Some(ttl)) + .await?; + Ok(fails > allowed) + } + + /// Count a success, and the tokens and request against the deployment's per-minute usage + /// when `tracks` says it has limits to track. The count is the success callback's; the usage + /// is `make_call`'s, which runs whatever the callbacks do. + pub async fn record_success( + &self, + callbacks: OutcomeCallbacks, + deployment: &Deployment, + total_tokens: u64, + tracks: bool, + ) -> Result<(), Error> { + callbacks + .record(OutcomeWrite::Success, self.count_success(&deployment.id)) + .await?; + if !tracks { + return Ok(()); + } + callbacks + .record( + OutcomeWrite::AttemptUsage, + self.add_usage(deployment, None, total_tokens as f64, 1.0), + ) + .await + } + + /// `_increment_deployment_usage`, for a deployment that tracks usage: tokens and requests + /// under `model` (metadata's `deployment`, which a wildcard route fills; else the + /// deployment's own), unless there is nothing to add. + pub async fn add_usage( + &self, + deployment: &Deployment, + model: Option<&str>, + total_tokens: f64, + requests: f64, + ) -> Result<(), Error> { + if total_tokens <= 0.0 && requests <= 0.0 { + return Ok(()); + } + let minute = current_minute(self.clock.now()); + let model = model.unwrap_or(&deployment.params.model); + self.cache + .async_increment_usage( + &[ + ( + usage_key(&deployment.id, model, UsageMetric::Tpm, &minute), + total_tokens, + ), + ( + usage_key(&deployment.id, model, UsageMetric::Rpm, &minute), + requests, + ), + ], + Some(USAGE_TTL), + ) + .await?; + Ok(()) + } + + /// `async_deployment_callback_on_failure`: a failed request counts against the deployment's + /// per-minute requests under `model` (see [`Self::add_usage`]), whether or not it tracks usage. + pub async fn count_failed_request( + &self, + deployment: &Deployment, + model: Option<&str>, + ) -> Result<(), Error> { + let minute = current_minute(self.clock.now()); + self.cache + .async_increment( + &usage_key( + &deployment.id, + model.unwrap_or(&deployment.params.model), + UsageMetric::Rpm, + &minute, + ), + 1.0, + Some(USAGE_TTL), + ) + .await?; + Ok(()) + } + + /// This minute's (tpm, rpm) for a deployment. + pub async fn usage(&self, deployment: &Deployment) -> Result<(u64, u64), Error> { + let minute = current_minute(self.clock.now()); + let model = &deployment.params.model; + let keys = [UsageMetric::Tpm, UsageMetric::Rpm] + .map(|metric| usage_key(&deployment.id, model, metric, &minute)); + let values = self.cache.async_batch_get_cache(&keys).await?; + let count = |index: usize| { + values + .get(index) + .cloned() + .flatten() + .and_then(|value| value.as_f64()) + .map_or(0, |count| count as u64) + }; + Ok((count(0), count(1))) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + + use super::*; + use crate::{ + cache::InMemoryRouterCache, + config::{RouterConfig, RouterDynamicConfig}, + registry::{DeploymentSpec, RoutingGroupSpec}, + }; + + #[rstest] + #[case::rate_limit(FailureKind::RateLimit, Some(429), true)] + #[case::auth(FailureKind::Authentication, Some(401), true)] + #[case::payment(FailureKind::Other, Some(402), true)] + #[case::not_found(FailureKind::NotFound, Some(404), true)] + #[case::timeout(FailureKind::Timeout, Some(408), true)] + #[case::bad_request(FailureKind::BadRequest, Some(400), false)] + #[case::forbidden(FailureKind::PermissionDenied, Some(403), false)] + #[case::server(FailureKind::InternalServer, Some(500), true)] + #[case::unreadable_status(FailureKind::Other, None, true)] + #[case::connection(FailureKind::ApiConnection, Some(500), false)] + fn cooldown_required_follows_python( + #[case] kind: FailureKind, + #[case] status: Option, + #[case] expected: bool, + ) { + assert_eq!( + is_cooldown_required(&AttemptError::new(kind, status)), + expected + ); + } + + /// An exception with no `status_code` is Python's status `""`, which no rule cools down. + #[rstest] + fn an_empty_status_requires_no_cooldown() { + let mut failure = AttemptError::new(FailureKind::Other, None); + failure.status_absent = true; + assert!(!is_cooldown_required(&failure)); + } + + #[rstest] + #[case::midnight(0, "00-00")] + #[case::afternoon(13 * 3600 + 7 * 60 + 59, "13-07")] + fn current_minute_is_utc_hours_and_minutes(#[case] seconds: u64, #[case] expected: &str) { + let time = UNIX_EPOCH + Duration::from_secs(86_400 * 3 + seconds); + assert_eq!(current_minute(time), expected); + } + + #[rstest] + #[case::content_policy_first(&["ContentPolicyViolationError", "BadRequestError"], Some(1.0))] + #[case::bad_request(&["BadRequestError"], Some(2.0))] + #[case::unnamed(&["Timeout"], None)] + fn a_deployment_policy_matches_in_python_order( + #[case] classes: &[&str], + #[case] expected: Option, + ) { + let policy = serde_json::json!({ + "BadRequestErrorAllowedFails": 2, + "ContentPolicyViolationErrorAllowedFails": 1, + }); + let mut failure = AttemptError::new(FailureKind::BadRequest, Some(400)); + failure.classes = classes.iter().map(|class| (*class).into()).collect(); + assert_eq!( + deployment_allowed_fails(policy.as_object(), &failure), + expected + ); + } + + fn deployment(id: &str, info: serde_json::Value) -> DeploymentSpec { + let mut model_info = serde_json::json!({"id": id}); + model_info + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + serde_json::from_value(serde_json::json!({ + "model_name": id, + "litellm_params": {"model": format!("openai/{id}")}, + "model_info": model_info, + })) + .unwrap() + } + + fn health() -> Health { + Health::new( + Arc::new(InMemoryRouterCache::default()), + Arc::new(SystemClock), + ) + } + + fn settings(dynamic: RouterDynamicConfig) -> Settings { + Settings::new(&RouterConfig::default(), Arc::new(dynamic)) + } + + #[rstest] + #[case::held(false, false, 1, true)] + #[case::caller_scoped(false, true, 1, false)] + #[case::outside_the_deployment(true, false, 0, false)] + #[tokio::test] + async fn the_failure_exits_decide_what_is_counted_and_cooled( + #[case] outside_deployment: bool, + #[case] caller_scoped: bool, + #[case] counted: u64, + #[case] cooled: bool, + ) { + let health = health(); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + let mut failure = AttemptError::new(FailureKind::RateLimit, Some(429)); + failure.outside_deployment = outside_deployment; + failure.caller_scoped = caller_scoped; + + let cooldown = health + .record_failure( + &settings(RouterDynamicConfig::default()), + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown.is_some(), cooled); + assert_eq!(health.outcomes_this_minute("a").await.unwrap().1, counted); + } + + #[rstest] + #[case::authentication( + FailureKind::CallerCredentialAuthentication, + 401, + "AuthenticationError" + )] + #[case::rate_limit(FailureKind::CallerCredentialRateLimit, 429, "RateLimitError")] + #[tokio::test] + async fn a_caller_credential_failure_counts_without_cooling( + #[case] kind: FailureKind, + #[case] status: u16, + #[case] base_class: &str, + ) { + let health = health(); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + let failure = AttemptError::new(kind, Some(status)); + + let cooldown = health + .record_failure( + &settings(RouterDynamicConfig::default()), + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown, None); + assert!(failure.is_a(base_class)); + assert_eq!(health.outcomes_this_minute("a").await.unwrap().1, 1); + } + + /// `litellm.allowed_fails` written after the build: the copied 3 starts counting instead of + /// the failure-ratio rule, which cools a 429 down at once. + #[rstest] + #[case::still_the_process_default(true, true)] + #[case::the_process_default_moved(false, false)] + #[tokio::test] + async fn the_copied_allowed_fails_counts_once_it_differs_from_the_process_default( + #[case] allowed_fails_is_default: bool, + #[case] cooled: bool, + ) { + let settings = settings(RouterDynamicConfig { + allowed_fails: Some(3), + allowed_fails_is_default, + ..RouterDynamicConfig::default() + }); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + let failure = AttemptError::new(FailureKind::RateLimit, Some(429)); + + let cooldown = health() + .record_failure( + &settings, + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown.is_some(), cooled); + } + + fn team(id: &str, team_id: &str, blocked: bool) -> DeploymentSpec { + deployment( + id, + serde_json::json!({"team_id": team_id, "team_public_model_name": "tp", "blocked": blocked}), + ) + } + + #[rstest] + #[case::a_sibling(team("b", "t1", false), true)] + #[case::a_blocked_sibling(team("b", "t1", true), false)] + #[case::another_teams_deployment(team("b", "t2", false), false)] + #[case::no_team(deployment("b", serde_json::json!({})), false)] + fn a_team_deployment_has_alternatives_under_its_public_name( + #[case] other: DeploymentSpec, + #[case] expected: bool, + ) { + let registry = Registry::build( + vec![team("a", "t1", false), other], + BTreeMap::new(), + Vec::new(), + ) + .unwrap(); + let first = registry.deployment("a").unwrap(); + assert_eq!(team_model_has_alternatives(®istry, &first), expected); + } + + #[rstest] + #[case::spans_two_deployments(&["a", "b"], "rg", true)] + #[case::one_deployment(&["a"], "rg", false)] + #[case::through_its_alias(&["a", "b"], "alias", true)] + #[case::a_member_called_directly(&["a", "b"], "a", false)] + fn a_routing_group_has_alternatives_past_one_deployment( + #[case] members: &[&str], + #[case] requested: &str, + #[case] expected: bool, + ) { + let group: RoutingGroupSpec = serde_json::from_value(serde_json::json!({ + "group_name": "rg", "models": members, "routing_strategy": "simple-shuffle", + })) + .unwrap(); + let registry = Registry::build( + vec![ + deployment("a", serde_json::json!({})), + deployment("b", serde_json::json!({})), + ], + BTreeMap::from([("alias".to_owned(), "rg".to_owned())]), + vec![group], + ) + .unwrap(); + assert_eq!( + routing_group_has_alternatives(®istry, requested), + expected + ); + } + + /// An empty status is counted and cools nothing down, unless the deployment's own policy + /// names the error. + #[rstest] + #[case::no_policy(serde_json::json!({}), false)] + #[case::policy_names_the_error( + serde_json::json!({"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}}), + true + )] + #[tokio::test] + async fn an_empty_status_on_the_attempt_path_needs_the_deployments_policy_to_cool( + #[case] info: serde_json::Value, + #[case] cooled: bool, + ) { + let health = health(); + let deployment = Deployment::from_spec(deployment("a", info)).unwrap(); + let mut failure = AttemptError::new(FailureKind::BadRequest, None); + failure.status_absent = true; + + let cooldown = health + .record_failure( + &settings(RouterDynamicConfig::default()), + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown.is_some(), cooled); + assert_eq!(health.outcomes_this_minute("a").await.unwrap().1, 1); + } + + #[rstest] + #[case::the_deployments_own_model(None, "openai/a")] + #[case::a_filled_wildcard(Some("openai/gpt-4o"), "openai/gpt-4o")] + #[tokio::test] + async fn usage_is_keyed_by_the_model_served(#[case] model: Option<&str>, #[case] keyed: &str) { + let cache = InMemoryRouterCache::default(); + let clock = Arc::new(SystemClock); + let health = Health::new(Arc::new(cache.clone()), clock.clone()); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + + health + .add_usage(&deployment, model, 15.0, 1.0) + .await + .unwrap(); + + let minute = current_minute(clock.now()); + assert_eq!( + ( + cache.get(&usage_key("a", keyed, UsageMetric::Tpm, &minute)), + cache.get(&usage_key("a", keyed, UsageMetric::Rpm, &minute)), + ), + (Some(serde_json::json!(15)), Some(serde_json::json!(1))), + ); + } + + #[rstest] + fn keys_match_python() { + assert_eq!( + ( + cooldown_key("abc"), + allowed_fails_key("abc", Some("generic")), + usage_key("abc", "openai/gpt-4o", UsageMetric::Tpm, "09-05"), + ), + ( + "deployment:abc:cooldown".to_owned(), + "deployment:abc:allowed_fails:generic".to_owned(), + "global_router:abc:openai/gpt-4o:tpm:09-05".to_owned(), + ) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/health_routing.rs b/litellm-rust/crates/router-core/src/health_routing.rs new file mode 100644 index 00000000000..bc3e43a6e92 --- /dev/null +++ b/litellm-rust/crates/router-core/src/health_routing.rs @@ -0,0 +1,126 @@ +//! Health-check-driven routing: the background health checks' results, kept in the shared +//! [`RouterCache`](crate::cache::RouterCache) under one key as Python's `DeploymentHealthCache` +//! writes them, and the filter `_async_filter_health_check_unhealthy_deployments` / +//! `_filter_health_check_unhealthy_deployments` run before the cooldown filter. + +use std::{collections::BTreeSet, sync::Arc, time::UNIX_EPOCH}; + +use serde_json::Value; + +use crate::{health::Health, registry::Deployment, settings::Settings}; + +/// `DeploymentHealthCache.CACHE_KEY`. +pub const HEALTH_STATE_KEY: &str = "litellm:health_check:deployment_health_state"; + +/// `_extract_unhealthy_ids`: the ids marked unhealthy whose result is younger than `staleness`. +pub fn unhealthy_ids(raw: Option<&Value>, now: f64, staleness: f64) -> BTreeSet { + let Some(Value::Object(states)) = raw else { + return BTreeSet::new(); + }; + states + .iter() + .filter(|(_, state)| { + let Value::Object(state) = state else { + return false; + }; + // `not state.get("is_healthy", True)`: any falsy value is unhealthy. + let unhealthy = match state.get("is_healthy") { + None => false, + Some(Value::Null) => true, + Some(Value::Bool(value)) => !value, + Some(Value::Number(value)) => value.as_f64() == Some(0.0), + Some(Value::String(value)) => value.is_empty(), + Some(Value::Array(value)) => value.is_empty(), + Some(Value::Object(value)) => value.is_empty(), + }; + let timestamp = state + .get("timestamp") + .and_then(Value::as_f64) + .unwrap_or(0.0); + unhealthy && now - timestamp < staleness + }) + .map(|(id, _)| id.clone()) + .collect() +} + +/// What the filter does when it would leave nothing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WhenAllUnhealthy { + /// The safety net: every candidate is kept. + KeepAll, + /// A health check probe sees no deployment. + Nothing, +} + +impl Health { + /// The deployments the background health checks have not marked unhealthy, as the + /// router's health-check routing settings say. + pub async fn health_check_filter( + &self, + settings: &Settings, + deployments: Vec>, + when_all_unhealthy: WhenAllUnhealthy, + ) -> Vec> { + if !settings.enable_health_check_routing { + return deployments; + } + let scoped = settings.background_health_check_model_groups.as_deref(); + // With an allowed_fails_policy cooldowns are the only exclusion, unless the filter is + // scoped to listed groups. + if settings.allowed_fails_policy.is_some() && scoped.is_none() { + return deployments; + } + // A read that fails counts as no health state. + let raw = self + .cache() + .async_get_cache(HEALTH_STATE_KEY) + .await + .ok() + .flatten(); + let now = self + .now_time() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs_f64()) + .unwrap_or_default(); + let unhealthy = unhealthy_ids( + raw.as_ref(), + now, + settings.health_check_staleness.as_secs_f64(), + ); + if unhealthy.is_empty() { + return deployments; + } + let filtered: Vec> = deployments + .iter() + .filter(|d| { + !unhealthy.contains(&d.id) + || scoped.is_some_and(|groups| !groups.contains(&d.model_name)) + }) + .cloned() + .collect(); + match (filtered.is_empty(), when_all_unhealthy) { + (false, _) => filtered, + (true, WhenAllUnhealthy::KeepAll) => deployments, + (true, WhenAllUnhealthy::Nothing) => Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::fresh_unhealthy(json!({"a": {"is_healthy": false, "timestamp": 95.0}}), vec!["a"])] + #[case::stale(json!({"a": {"is_healthy": false, "timestamp": 50.0}}), vec![])] + #[case::healthy(json!({"a": {"is_healthy": true, "timestamp": 95.0}}), vec![])] + #[case::no_flag(json!({"a": {"timestamp": 95.0}}), vec![])] + #[case::not_a_dict(json!({"a": 1}), vec![])] + fn unhealthy_ids_follow_python(#[case] raw: Value, #[case] expected: Vec<&str>) { + let ids = unhealthy_ids(Some(&raw), 100.0, 10.0); + assert_eq!(ids.iter().map(String::as_str).collect::>(), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/host.rs b/litellm-rust/crates/router-core/src/host.rs new file mode 100644 index 00000000000..0a72be8f980 --- /dev/null +++ b/litellm-rust/crates/router-core/src/host.rs @@ -0,0 +1,726 @@ +//! [`Host`]: the router core's only boundary. Everything the core needs from the embedding +//! process goes through it, and every attempt runs through [`Host::invoke`]: the core never runs +//! an operation itself. `PythonHost` serves it in a Python process (attempts call `litellm.`, +//! and the deployment build asks it for secrets, validation, providers and side effects). + +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::{ + cache::RouterCache, + config::{Op, RetryPolicy, RoutingStrategy}, + error::{AttemptError, Error, FailureKind, Payload}, + fallback::Fallbacks, + health::Health, + mock::MockFailure, + outcome::OutcomeCallbacks, + precall::PreCallFacts, + prerouting::{PreRoute, PreRouted}, + registry::{Deployment, Registry}, + settings::Settings, + strategy::StrategyScope, +}; + +/// One call into the router. +#[derive(Clone, Debug, Default)] +pub struct RouterRequest { + /// The requested name: a model group, alias, routing group, deployment id or wildcard match. + pub model: String, + /// The request params, for the plugins and host steps that read them. + pub body: Value, + /// `user_api_key_team_id`, from the first metadata bucket that holds one + /// (`get_request_team_id`). + pub team_id: Option, + /// The `metadata` bucket's `user_api_key_team_id` alone: the team a failed level lists the + /// group's deployments for (`async_function_with_fallbacks_common_utils`, + /// `_stream_order_levels`). `None` lists every team's. + pub listing_team_id: Option, + /// The caller's `user_role` is `proxy_admin`. + pub proxy_admin: bool, + /// The caller's `models` plus `team_models` (`user_api_key_auth`), which narrow a caller + /// allowed only through access groups to those groups' deployments. + pub allowed_models: Vec, + /// For a proxy admin, the model the request names (`model`, else the metadata + /// `model_group`), which keeps one team's deployments listed under it (`filter_team_based_models`). + pub admin_model: Option, + pub specific_deployment: bool, + /// A per-request `num_retries`, which wins over the router's. + pub num_retries: Option, + /// A per-request `num_retries` that is no count, which replaces the router's as well. + pub uncounted_retries: Option, + pub disable_fallbacks: bool, + /// `_router_weights` as the request passed it: model group to deployment id to weight, + /// validated when simple-shuffle picks ([`crate::weights::validate`]). + pub router_weights: Option, + /// The deployment a call on an existing resource (a video id) names: picked whenever the + /// filters keep it (`_narrow_to_video_deployment`). + pub pinned_deployment_id: Option, + /// A per-request `routing_strategy` override, as Python normalizes it. + pub routing_strategy: Option, + /// Whether tag routing may apply to this request (router, request or model setting), so the + /// host's tag stage runs. + pub filter_tags: bool, + /// A pass-through pick: only `use_in_pass_through` deployments may serve it. + pub pass_through: Option, + /// `_target_order`: the order level to pick from instead of the lowest. + pub target_order: Option, + /// `_excluded_deployment_ids`: never picked, even when that leaves nothing. + pub excluded_ids: Vec, + /// `_retry_skipped_deployment_ids`: skipped unless that leaves nothing. + pub retry_skipped_ids: Vec, + /// The endpoint `endpoint_selection` names around the call, which deployments that declare + /// `enforce_supported_endpoints` may refuse ([`crate::eligibility`]). + pub endpoint: Option, + /// A web search request on an async pick: `filter_web_search_deployments` drops the + /// deployments that declare no support. The sync pick never runs that filter. + pub web_search: bool, + /// The handler name of a call only some providers serve (`_ROUTE_CAPABILITY_RESOLVERS`): + /// the deployments whose provider cannot serve it are dropped unless that leaves nothing. + pub route_handler: Option, + /// A Decisions call carrying audio or videos: the deployments that refuse them are skipped + /// unless that leaves nothing (`_without_decisions_deployments_refusing_media`). + pub decisions_media: bool, + /// `f"{provider}/{model}"` for a lookup that cannot ask the host + /// (`_core_get_routable_deployments`); a pick asks [`Host::provider_request`]. + pub wildcard_request: Option, + /// The request carries an image (`_request_carries_image`), so deployments that declare no + /// vision are skipped while any other remains. + pub has_image: bool, + /// An async pick: the host's pre-routing step runs before each pick resolves its name. + pub pre_routes: bool, + /// Which of the router's outcome callbacks run for this request. + pub outcome_callbacks: OutcomeCallbacks, + /// The requested name is no model group (a search tool name): nothing is selected, every + /// attempt is handed to the host as is, and no outcome is recorded for a deployment + /// (`_asearch_with_fallbacks`, whose helper picks among the router's search tools). + pub unrouted: bool, + /// `passthrough_on_no_deployment`: a pick that fails hands the attempt to the host under the + /// requested name, inside the level's retries and fallbacks, with no outcome recorded for a + /// deployment (`_ageneric_api_call_with_fallbacks_helper`). + pub passthrough_on_no_deployment: bool, + /// The endpoint's attempt stamps no `failed_deployment_id` on its error (file and batch + /// creation, whose result belongs to the deployment that served it), so weighted failover + /// never runs for it. + pub skip_weighted_failover: bool, + /// The request creates or names a resource only one provider holds (a file upload, a batch + /// on an uploaded file): it falls back within its own model group only + /// (`run_async_fallback`'s `same_model_group_only`). + pub same_group_fallbacks_only: bool, + /// Fallback lists the request passed (`Some(None)` when it passed `None`), which replace + /// the router's. + pub fallbacks: Option>, + pub context_window_fallbacks: Option>, + pub content_policy_fallbacks: Option>, + /// A per-request `max_fallbacks`, which replaces the router's. + pub max_fallbacks: Option, + /// A per-request `cooldown_time`: the call's own params carry it, so it cools a failed + /// deployment down for that long unless its `model_info` sets one. + pub cooldown_time: Option, + /// A `mock_testing_*` flag the request set. + pub mock_failure: Option, + /// A per-request `model_group_retry_policy`, which replaces the router's. + pub model_group_retry_policy: Option>, + /// The model groups whose `model_group_retry_policy` lookup raises (the policy is no mapping + /// of policies, or the group's entry is not one): the first failure of a hop serving one ends + /// the hop with the host's error instead ([`RouterEvent::RetryPolicyRaised`]), no retry. + pub retry_policy_raises: Vec, + /// `health_check_probe`: a health check's own pick, which sees no deployment when every + /// one is marked unhealthy (the safety net keeps them all for any other request). + pub health_check_probe: bool, + /// What the host needs to run the call itself, such as the Python kwargs. + pub payload: Option, +} + +/// A per-request `num_retries` that is no count of retries. `async_function_with_retries` only +/// reads it once a hop's first failure was not taken over by a retry policy: a call that succeeds +/// first time never does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UncountedRetries { + /// A negative int: `num_retries > 0` is false, so the failure stands, and a stream's plain + /// budget is spent (`isinstance(num_retries, int)`). + Negative, + /// Any other value `num_retries > 0` is false for (a negative float, a NaN): the failure + /// stands, and a stream falls through to the deployment's budget, then the router's. + NotPositive, + /// `num_retries > 0` or `range(num_retries)` raises (a string, a positive float): the hop ends + /// with that error ([`RouterEvent::RetryCountRaised`]). + Raises, +} + +/// One `os.environ/` reference the deployment build found ([`crate::build`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SecretRef { + /// The value as written, `os.environ/`. + pub name: String, + /// Where the value sits in the deployment. + pub at: SecretAt, +} + +/// Where a [`SecretRef`] sits in a deployment. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SecretAt { + /// A top-level `litellm_params` value: resolved through the host's secret manager. + Param(String), + /// `litellm_params.dataSources[source].parameters[key]` (`endpoint` or `key`): read from the + /// environment alone, empty when unset. + DataSource { source: usize, key: String }, +} + +/// Which of the router's adds a build runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum BuildPath { + /// `set_model_list` (and the constructor's list): config entries, one deployment per + /// organization, ignored when invalid if the router ignores invalid deployments. + Create, + /// `add_deployment` (and upserts): one validated deployment. + Add, +} + +/// What the build asks the host about one deployment. +#[derive(Clone, Debug)] +pub struct BuildAsk<'a> { + pub path: BuildPath, + /// The entry's position in the list the build was given. + pub index: usize, + /// The entry as the build holds it: `model_name`, `litellm_params` (secrets resolved, the + /// organization of this deployment set), `model_info` (with its `id`) and any other keys. + pub entry: &'a Map, + /// The `model_info.id` the entry declared, before the build assigned one. + pub declared_id: Option<&'a str>, + /// Another entry of the same list declares the same id. + pub duplicate_id: bool, +} + +/// The host's answer about one deployment. +#[derive(Clone, Debug, PartialEq)] +pub enum Built { + Entry(Map), + /// Left out without an error (`model_info.supported_environments` excludes this one, or an + /// `add_deployment` id that already exists). + Skip, + /// The host failed the deployment and the router ignores invalid deployments. + Invalid, +} + +/// The host's provider answer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Provided { + Provider(Option), + Invalid, +} + +/// What the model-info merge ([`crate::model_info`]) asks the host while it answers one query. +/// Asked at query time, so a price reload or a registration the host made since the router was +/// built is what the next query reads. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(tag = "lookup", rename_all = "snake_case")] +pub enum ModelInfoLookup { + /// The process cost map's row for `model` (`get_model_info(model=...)`), the host choosing + /// the keys it tries. Answered with the row, or `None`. A lookup that raises propagates, + /// unless `tolerate` is set: then the host answers `{"failed": True}`. `exact` also asks + /// whether the row came from a fallback-generalization rule (`{"row": ..., "generalized": + /// bool}`). + ModelInfo { + model: Value, + tolerate: bool, + exact: bool, + }, + /// The raw cost-map entry under `key` (`model_cost.get(key)`): answered with + /// `{"entry": value}`, or `None` when the key is absent. `process` reads the process global + /// (`litellm.model_cost`) rather than the host's live settings, where the Python router does. + CostEntry { key: String, process: bool }, + /// `get_llm_provider` for the queried deployment's `model` and validated params: answered + /// with `[model, provider]`. + Provider { model: String }, + /// The deployments a wildcard pattern routes `name` to: answered with `[{id, model}]`, or + /// `None` when no pattern matches. + PatternModels { name: String }, + /// The base model a bedrock model id names (`get_bedrock_base_model`). + BedrockBase { model_id: String }, + /// The limits the background model-info refresh discovered for deployment `id`. + Discovered { id: Value }, + /// The queried deployment's own `model_info`, read when the merge reaches it (a malformed one + /// raises there). + UserModelInfo, + /// A log record, emitted where the Python router logs it. + Log { level: LogLevel, message: String }, + /// Group info: deployment `index`'s validated params (`LiteLLM_Params(**litellm_params)`, which + /// raises on bad params): answered with `{"model", "configurable_clientside_auth_params"}`. + GroupParams { index: usize }, + /// Group info: what the host knows of deployment `index` serving `model`: answered with + /// `{"model_info"}` (its merged model info, `None` when that fails), `"mapped"` + /// (`deployment_is_catalog_mapped`), `"litellm_model"` and `"llm_provider"`, and, when it has + /// no model info, `"supported_openai_params"`. + GroupDeployment { index: usize, model: Value }, + /// Group info: deployment `index`'s capabilities under `model_info`: answered with + /// `{"supported_endpoints"}` (empty when the team may not use it), `"supports_fast_mode"` and + /// `"reasoning_efforts"` (`resolve_supported_reasoning_efforts`, or `None`). + GroupCapabilities { + index: usize, + model_info: Value, + litellm_model: Value, + llm_provider: Value, + mapped: bool, + }, + /// Group info: the group's first deployment as the host builds it (`ModelGroupInfo(**info)`, + /// whose validation may raise): answered with the fields the merge compares, as validated. + GroupSeed { info: Map }, + /// `intersect_supported_reasoning_efforts(current, resolved)`, in the host's advertisement order. + ReasoningIntersect { current: Value, resolved: Value }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LogLevel { + Debug, + Error, +} + +/// One attempt against one deployment. +#[derive(Clone, Debug)] +pub struct Attempt { + pub op: Op, + pub request: Arc, + pub deployment: Arc, + /// The name this attempt resolved: the requested name, or the fallback target. + pub name: String, + /// The model group being served, after fallbacks. + pub model_group: String, + /// The `order` level an order fallback targets in the same group. + pub target_order: Option, + /// Every deployment `name` resolved to for this caller, the picked one included. + pub candidate_ids: Vec, + /// 0 for the first attempt in this group, then the retry number. + pub retry: u32, + pub fallback_depth: u32, + /// How many fallbacks deep the chain is: 0 for the requested name. + pub fallback_level: u32, + pub timeout: Option, + /// Request params the fallback chain's dict targets merged in (`kwargs.update(target)`). + pub params: Option>>, + /// The chain level whose request metadata this attempt carries (see [`RouterEvent::Fallback`]). + pub scope: u32, + /// The retry budget the level's loop stamps on the request before this attempt. + pub max_retries: u32, + /// How many deployments `get_model_list` lists for the level's group (`model_group_size`). + pub model_group_size: usize, +} + +/// What a successful attempt returns. +#[derive(Clone, Debug)] +pub struct Output { + pub value: Payload, + /// Prompt plus completion tokens, counted against the deployment's per-minute usage. + pub total_tokens: u64, + /// A response the provider blocked, which the router may raise to fall back. + pub blocked: Option, + /// A stream the host reports the outcome of when it ends ([`crate::Router::report_success`], + /// [`crate::Router::report_failure`]), or a realtime session, whose success the host counts + /// only when it ended cleanly: the attempt counts its request and `total_tokens` now, and no + /// success. + pub deferred: bool, + /// Whether the attempt counts against the deployment's per-minute usage, when the host knows + /// better than the deployment's own limits ([`Deployment::tracks_usage`]): a limit only the + /// model catalog sets (`_deployment_tracks_usage`), or a realtime session, whose `None` + /// response counts nothing (`increment_deployment_usage_for_response`). + pub tracks_usage: Option, +} + +/// How a provider blocked a response the call otherwise served. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum Blocked { + /// A chat response whose first choice was content-filtered (or that has no choices). + ContentFilter, + /// An Anthropic /v1/messages safeguard refusal carrying `stop_details`. + Refusal, +} + +/// What a plugin or strategy sees: never the router itself. +pub struct RoutingContext<'a> { + pub registry: &'a Registry, + pub settings: &'a Settings, + pub health: &'a Health, + pub request: &'a RouterRequest, + pub op: &'a Op, + pub model_group: &'a str, + /// The name the request resolved from (Python's `model`), which strategies key state by. + pub name: &'a str, + /// Whose selector a strategy stage reads. + pub scope: StrategyScope, + /// The deployments the name resolved to for this caller, before health filters + /// (`request_deployments`), for the tag stage. + pub pool: &'a [Arc], + /// `_excluded_deployment_ids` a weighted failover hop put on the request, which the callback + /// filters read (`deployment_ids_failed_in_request`). + pub excluded_ids: &'a [String], + /// `_retry_skipped_deployment_ids` the retry loop put on the request, as `excluded_ids`. + pub skipped_ids: &'a [String], + /// `_target_order` as the filters read it: the level an order fallback targets, else the one + /// the request named. + pub target_order: Option, +} + +/// Which selection step the host is asked to run. +#[derive(Clone, Debug, PartialEq)] +pub enum SelectStage { + /// Pre-call checks and sub-routers registered as plugins: return the deployments that remain. + Plugins, + /// A strategy not yet ported: return the candidates with the pick first. + Strategy(RoutingStrategy), + /// Tag routing (`get_deployments_for_tag`), not yet ported: return the deployments that remain. + Tags, + /// `_deployment_declares_no_vision`: return the candidates that declare no vision support. + TextOnly, + /// `_deployment_serves_route`: return the candidates whose provider cannot serve the call + /// `handler` names (a Decisions call on a provider with no Decisions config). + RouteIncapable { handler: String }, + /// `_decisions_deployment_refuses_media`: return the candidates that refuse the audio or + /// videos a Decisions call carries. + MediaRefusing, +} + +/// Routing events, delivered through [`Host::observe`]. +#[derive(Clone, Debug, PartialEq)] +pub enum RouterEvent { + Selected { + model_group: String, + deployment_id: String, + /// How long the pick took (`_acompletion`'s `async_get_available_deployment` service event). + pick: Duration, + }, + Succeeded { + deployment_id: String, + }, + AttemptFailed { + deployment_id: String, + kind: FailureKind, + status: Option, + }, + CooldownSet { + deployment_id: String, + cooldown: Duration, + }, + Retry { + model_group: String, + retry: u32, + delay: Duration, + }, + Fallback { + from: String, + to: String, + /// The order level of `to` a same-group hop walks to. + order: Option, + depth: u32, + /// The fallback level `to` is tried at: 1 for the request's own fallbacks. + level: u32, + /// The new level's request metadata scope, copied from `parent`'s at the hop. + scope: u32, + parent: u32, + /// The `max_fallbacks` the chain reads, stamped on the hop's request. + max_fallbacks: u32, + /// The non-string `model` of the dict target this hop routes + /// ([`crate::fallback::FallbackTarget::unroutable_model`]): the hop fails on it before any + /// attempt, and the host raises what Python's level raises for that value. + unroutable: Option, + }, + /// A hop of the fallback chain returned: `log_success_fallback_event` or + /// `log_failure_fallback_event` for the level at `parent`. + FallbackDone { + from: String, + to: String, + level: u32, + scope: u32, + parent: u32, + succeeded: bool, + }, + /// The router raised a `ContentPolicyViolationError` for a blocked response, since a + /// fallback would serve it (`_should_raise_content_policy_error`, + /// `_should_raise_anthropic_refusal_error`). + Blocked { + level: u32, + model: String, + blocked: Blocked, + }, + /// `log_retry`: the router recorded a failure in `scope`'s request metadata + /// (`previous_models`, `request_retry_count`) before retrying or falling back. + RetryLogged { + scope: u32, + level: u32, + /// The group the failed level served (`kwargs["model"]`). + model: String, + /// The retry number stamped before the failed try; `None` keeps the scope's own. + attempted_retries: Option, + cause: RetryCause, + }, + /// `async_function_with_retries` opened a level's retry loop, before any pick: `scope`'s + /// request metadata tracks no retry yet, `max_retries` of them, and the group's size. + RetriesOpened { + scope: u32, + max_retries: u32, + model_group_size: usize, + }, + /// The level's first failure settled its overrides: the budget `scope` then tracks. + RetryBudget { + scope: u32, + max_retries: u32, + }, + /// A pick of the async selection or the sync one failed on `error`: the async one's `except` + /// logs every refusal to the request's logging object (`_async_get_available_deployment`). + PickFailed { + level: u32, + model: String, + error: PickError, + }, + /// A level of the fallback chain serving `model` ended on an error a pick raised (no + /// deployment for the caller, none available, a stale routing group, ...): Python raises + /// that error for the level, whatever its earlier attempts raised. + LevelFailed { + level: u32, + model: String, + error: PickError, + /// Raised by the level's fallback handling, while the level's own error was handled + /// (so it chains to that error), rather than by the level's retry loop. + handling: bool, + }, + /// The router raised a `mock_testing_*` failure for `model` instead of calling it. + MockFailed { + mock: MockFailure, + level: u32, + model: String, + num_retries: Option, + }, + /// The first failure of a hop serving `model` looked up a `model_group_retry_policy` that + /// raises for it: the hop ends with the error the lookup raised (the host raises it, from the + /// failure `cause` names), without retrying, and falls back as from any plain exception. + RetryPolicyRaised { + level: u32, + model: String, + cause: RetryCause, + }, + /// The first failure of a hop serving `model` read a request `num_retries` that raises + /// ([`UncountedRetries::Raises`]): the hop ends with that error (the host raises it, from the + /// failure `cause` names), and falls back as from any plain exception. + RetryCountRaised { + level: u32, + model: String, + cause: RetryCause, + }, + /// The level serving `model` reached the fallback handling that appends the context window + /// and content policy hints to its error, before any hop runs + /// (`async_function_with_fallbacks_common_utils`). The order path never gets there. + FallingBack { + level: u32, + model: String, + }, + /// The order fallback chain of the level at `level` served nothing, so the level raises from + /// `async_function_with_fallbacks_common_utils`' order path, with no outcome message: what the + /// chain's last hop raised when `last_hop`, else its own error (a context window error, or a + /// chain no hop of which ran). Weighted failover raising what it does not catch + /// (`_maybe_run_weighted_failover`) ends the level the same way. + OrderChainFailed { + level: u32, + last_hop: bool, + }, + /// The regular fallback handling of the level at `level` served nothing, where + /// `async_function_with_fallbacks_common_utils` reaches its `except` and logs the failure. + /// `chain` when the level had targets to walk: the error it logs is what the last hop raised + /// when `last_hop`, else its own. With no `chain`, the host decides: only the host holds + /// whether a level with no match had a fallback list at all, which Python does not log. + /// The order path and weighted failover are outside that `try` and never report this. + FallbacksFailed { + level: u32, + chain: bool, + last_hop: bool, + }, + /// A group's retry loop ran every retry and the last one failed: Python stamps + /// `num_retries` and `max_retries` on that error. + RetriesExhausted { + model_group: String, + attempted: u32, + max: u32, + /// The level the retry loop served. + level: u32, + /// The last failure was a pick's error, which the level then raises + /// ([`RouterEvent::LevelFailed`]), rather than an attempt's. + picked: bool, + }, +} + +/// A pick's error, as the events that name it carry it. +#[derive(Clone, Debug)] +pub struct PickError(pub Arc); + +impl PickError { + /// The copy of `error` events carry, when a pick raised it. + pub fn of(error: &Error) -> Option { + error.pick_copy().map(|copy| Self(Arc::new(copy))) + } +} + +impl PartialEq for PickError { + fn eq(&self, other: &Self) -> bool { + format!("{:?}", self.0) == format!("{:?}", other.0) + } +} + +/// Which error a [`RouterEvent::RetryLogged`] records. +#[derive(Clone, Debug, PartialEq)] +pub enum RetryCause { + /// The last attempt's failure. + Attempt, + /// The error the level ended on (`run_async_fallback`'s `original_exception`). + Level, + /// A pick raised this error: the group was unknown or had none available, the stale + /// routing-group member map named a group no longer configured, ... + Pick(PickError), + /// Any other error no attempt raised. + Other, +} + +impl RetryCause { + /// The cause that names `error`. + pub fn of(error: &Error) -> Self { + match error { + Error::Attempt(_) => Self::Attempt, + _ => PickError::of(error).map_or(Self::Other, Self::Pick), + } + } +} + +#[async_trait] +pub trait Host: Send + Sync { + /// Run one attempt against the picked deployment. The core records its outcome (cooldowns, + /// allowed-fails counters, usage) itself; the host only runs it. + async fn invoke(&self, attempt: &Attempt) -> Result; + + /// Run a selection stage the core does not implement. + async fn select( + &self, + stage: &SelectStage, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error>; + + /// Pre-routing before a pick resolves `ask.name`: the model group to resolve instead, a + /// custom strategy's deployment to serve, or `None` to resolve the name itself. Asked only + /// for picks that pre-route (`pre_routes`). + async fn pre_route(&self, ask: &PreRoute<'_>) -> Result, Error> { + let _ = ask; + Ok(None) + } + + /// Access and budget checks before a cross-group fallback target. + /// `Err` when a check raised: the chain ends there, outside any target's `try`. + async fn allow_fallback( + &self, + request: &RouterRequest, + target: &str, + ) -> Result; + + /// Whether `run_async_fallback` refuses a chain for one of its dict `targets`, a server-owned + /// federation field or an `oidc/` api key. The chain is walked by `level`. + async fn refuses_fallbacks(&self, _level: u32, _targets: &[Map]) -> bool { + false + } + + async fn sleep(&self, duration: Duration); + + fn observe(&self, event: &RouterEvent); + + /// The shared store routing state lives in. + fn cache(&self) -> Arc; + + /// Whether [`Self::invoke`] can run `op` on `provider`. Checked for every served op and + /// deployment when the router is built or changed, so an unsupported pair fails the config, + /// never a request. `PythonHost` runs every op. + fn supports(&self, _op: &Op, _provider: &str) -> bool { + true + } + + /// Whether [`Self::select`] can run strategies and plugins. + fn can_select(&self) -> bool { + true + } + + /// Run the routing plugins for `ctx.name` when `run` (the async pick's pre-routing hook), and + /// report the candidate models they left on the request, if any. + async fn routing_plugin_models( + &self, + _ctx: &RoutingContext<'_>, + _run: bool, + ) -> Result>, Error> { + Ok(None) + } + + /// The facts the `enable_pre_call_checks` filter decides by (token count, context windows, + /// param support), for `candidates` and the deployments in `ctx.pool`. A host without them + /// reports no input, so nothing is checked. + async fn pre_call_facts( + &self, + _ctx: &RoutingContext<'_>, + _candidates: &[Arc], + ) -> Result { + Ok(PreCallFacts::default()) + } + + /// `f"{provider}/{model}"` for `ctx.name`, which `PatternMatchRouter.get_pattern` tries when + /// no pattern matches the name itself: only the host can infer a provider. `None` from a + /// host that infers none. + async fn provider_request(&self, _ctx: &RoutingContext<'_>) -> Result, Error> { + Ok(None) + } + + /// `async_callback_filter_deployments`: the host's routing callbacks narrow `candidates` in + /// turn. A filter that raises fails the try with [`Error::Attempt`], which the call retries and + /// falls back from like any attempt failure. A host without filters keeps every candidate. + async fn callback_filters( + &self, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + Ok(candidates) + } + /// The values of `refs`, one per ref, in order: what the deployment build substitutes for + /// each `os.environ/` reference it found in entry `index` ([`crate::build`]). Asked once per + /// deployment, as the build reaches it. A host without secrets leaves every reference as + /// written. + async fn secrets(&self, index: usize, refs: &[SecretRef]) -> Result, Error> { + let _ = index; + Ok(refs.iter().map(|r| Value::String(r.name.clone())).collect()) + } + + /// Check one deployment before it is added, and answer the entry as the host normalized it, + /// a skip (the deployment is left out), or [`Built::Invalid`] (it failed and the router + /// ignores invalid deployments). A host without checks accepts the entry as given. + async fn validate(&self, ask: &BuildAsk<'_>) -> Result { + Ok(Built::Entry(ask.entry.clone())) + } + + /// The provider that serves one validated deployment (`get_llm_provider`), `None` for one + /// served without a provider (a prompt-management model), or [`Provided::Invalid`]. A host + /// without provider resolution answers `None`, so [`Deployment::provider`] reads the model. + async fn provider(&self, ask: &BuildAsk<'_>) -> Result { + let _ = ask; + Ok(Provided::Provider(None)) + } + + /// Run the host's side effects for an accepted deployment and answer the entry as the + /// registry stores it. A host without side effects stores the entry it validated. + async fn added(&self, ask: &BuildAsk<'_>, provider: Option<&str>) -> Result { + let _ = provider; + Ok(Built::Entry(ask.entry.clone())) + } + + /// Answer one model-info lookup ([`ModelInfoLookup`]). A host without a cost map answers + /// `None` to everything, so no deployment has model info. + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + let _ = lookup; + Ok(Value::Null) + } +} diff --git a/litellm-rust/crates/router-core/src/host_index.rs b/litellm-rust/crates/router-core/src/host_index.rs new file mode 100644 index 00000000000..3748f7b4d19 --- /dev/null +++ b/litellm-rust/crates/router-core/src/host_index.rs @@ -0,0 +1,727 @@ +//! [`HostIndexes`]: the registry indexes the Python router hands out, kept the way it keeps them. +//! +//! The Python router keeps its indexes by history rather than deriving them from the deployment +//! list: `model_names` is the set of names `set_model_list` was given (invalid entries included), +//! `deployment_names` only ever grows, assigning `model_list` reindexes nothing, a removal shifts +//! the positions every index holds (a caller-written one included), and the wildcard pattern +//! routers keep the copy of a deployment they were given. The registry keeps one of these per +//! version and applies those rules on its own writes; the host reads them back as the live +//! objects `registry().index()`, `names()` and `patterns()` return, and hands a caller's write to +//! any of them back as a changeset ([`IndexWrite`]). +//! +//! Values are JSON so whatever a caller wrote reads back as written, junk included. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// One wildcard pattern router (`PatternMatchRouter`): regex to the deployments it routes to, +/// most specific pattern first. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct PatternRouter { + pub patterns: Vec<(String, Vec)>, +} + +/// The indexes, as the host reads and writes them. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct HostIndexes { + /// `model_id_to_deployment_index_map`: id to position, in insertion order. + pub id_map: Map, + /// `model_name_to_deployment_indices`: name to its positions. + pub model_name: Map, + /// `team_model_to_deployment_indices`: `[team_id, public name, positions]`. + pub team_model: Vec<(String, String, Value)>, + /// `model_names`: a set, its members in the order they joined. + pub model_names: Vec, + pub deployment_names: Vec, + pub team_public_model_names: Vec, + pub provider_default_ids: Vec, + /// `pattern_router`. + pub patterns: PatternRouter, + /// `team_pattern_routers`: team id to its router. + pub team_patterns: Vec<(String, PatternRouter)>, +} + +/// A host write to the indexes: each field it names replaces the one held. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default)] +pub struct IndexWrite { + pub id_map: Option>, + pub model_name: Option>, + pub team_model: Option>, + pub model_names: Option>, + pub deployment_names: Option>, + pub team_public_model_names: Option>, + pub provider_default_ids: Option>, + pub patterns: Option, + pub team_patterns: Option>, +} + +/// A removal the Python router could not finish: an index held a position it cannot compare +/// (`_without_indices` raised `TypeError`). The indexes keep what it shifted before that. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("'>' not supported between instances of '{found}' and 'int'")] +pub struct RemovalError { + pub found: &'static str, +} + +impl HostIndexes { + /// Apply a host write. + pub fn write(&mut self, write: IndexWrite) { + let IndexWrite { + id_map, + model_name, + team_model, + model_names, + deployment_names, + team_public_model_names, + provider_default_ids, + patterns, + team_patterns, + } = write; + if let Some(value) = id_map { + self.id_map = value; + } + if let Some(value) = model_name { + self.model_name = value; + } + if let Some(value) = team_model { + self.team_model = value; + } + if let Some(value) = model_names { + self.model_names = unique(value); + } + if let Some(value) = deployment_names { + self.deployment_names = value; + } + if let Some(value) = team_public_model_names { + self.team_public_model_names = value; + } + if let Some(value) = provider_default_ids { + self.provider_default_ids = value; + } + if let Some(value) = patterns { + self.patterns = value; + } + if let Some(value) = team_patterns { + self.team_patterns = value; + } + } + + /// `set_model_list` starting over: the id, name and team indexes and the team routers go; + /// the deployment names, the wildcard router and the provider default ids stay. + pub fn reset(&mut self) { + self.id_map.clear(); + self.model_name.clear(); + self.team_model.clear(); + self.team_patterns.clear(); + self.team_public_model_names.clear(); + } + + /// `set_model_list`'s `model_names`: the names it was given. + pub fn set_model_names(&mut self, names: Vec) { + self.model_names = unique(names.into_iter().map(Value::String).collect()); + } + + /// A deployment joined the list at `position` (`_add_deployment` then + /// `_add_model_to_list_and_index_map`); `add` is `add_deployment`, which also names its group. + pub fn added(&mut self, entry: &Map, position: usize, add: bool) { + let params = entry.get("litellm_params").and_then(Value::as_object); + let info = entry.get("model_info").and_then(Value::as_object); + let model_name = entry.get("model_name").and_then(Value::as_str); + let id = info + .and_then(|info| info.get("id")) + .filter(|id| !id.is_null()); + self.deployment_names.push( + params + .and_then(|params| params.get("model")) + .cloned() + .unwrap_or(Value::Null), + ); + if let Some(name) = model_name.filter(|name| name.contains('*')) { + self.patterns.add(name, Value::Object(entry.clone())); + if let Some(id) = id.and_then(Value::as_str).filter(|id| !id.is_empty()) { + self.provider_default_ids.push(id.to_owned()); + } + } + let team_id = info + .and_then(|info| info.get("team_id")) + .filter(|v| !v.is_null()); + let public = info + .and_then(|info| info.get("team_public_model_name")) + .filter(|v| !v.is_null()); + if let (Some(team_id), Some(Value::String(public))) = (team_id, public) + && public.contains('*') + { + let team = python_str(team_id); + let router = match self.team_patterns.iter_mut().find(|(t, _)| *t == team) { + Some((_, router)) => router, + None => { + self.team_patterns.push((team, PatternRouter::default())); + &mut self.team_patterns.last_mut().expect("just pushed").1 + } + }; + router.add(public, Value::Object(entry.clone())); + } + let position = Value::from(position); + if let Some(id) = id { + self.id_map.insert(python_str(id), position.clone()); + } + if let Some(name) = model_name.filter(|name| !name.is_empty()) { + push_position( + self.model_name + .entry(name.to_owned()) + .or_insert_with(|| Value::Array(Vec::new())), + &position, + ); + } + if add { + let name = Value::String(model_name.unwrap_or_default().to_owned()); + if !self.model_names.contains(&name) { + self.model_names.push(name); + } + } + // `_update_team_model_index`: a truthy team id and public name. + if let (Some(team_id), Some(public)) = (team_id, public) + && truthy(team_id) + && truthy(public) + { + let (team, public) = (python_str(team_id), python_str(public)); + if !self.team_public_model_names.contains(&public) { + self.team_public_model_names.push(public.clone()); + } + let held = match self + .team_model + .iter_mut() + .find(|(t, p, _)| *t == team && *p == public) + { + Some((_, _, held)) => held, + None => { + self.team_model + .push((team, public, Value::Array(Vec::new()))); + &mut self.team_model.last_mut().expect("just pushed").2 + } + }; + if let Value::Array(positions) = held + && !positions.contains(&position) + { + positions.push(position); + } + } + } + + /// The deployment `id` left the list from `position` (`_update_deployment_indices_after_removal`). + pub fn removed(&mut self, id: &str, position: usize) -> Result<(), RemovalError> { + let mut id_map = Map::new(); + for (key, held) in std::mem::take(&mut self.id_map) { + if key == id { + continue; + } + let shifted = shift(&held, position)?.unwrap_or(held); + id_map.insert(key, shifted); + } + self.id_map = id_map; + let names: Vec = self.model_name.keys().cloned().collect(); + for name in names { + let kept = without(&self.model_name[&name], position)?; + if kept.is_empty() { + self.model_name.remove(&name); + self.model_names + .retain(|held| held.as_str() != Some(name.as_str())); + } else { + self.model_name.insert(name, Value::Array(kept)); + } + } + let mut team_model = Vec::new(); + let held = std::mem::take(&mut self.team_model); + let mut rest = held.into_iter(); + for (team, public, positions) in rest.by_ref() { + match without(&positions, position) { + Ok(kept) if kept.is_empty() => {} + Ok(kept) => team_model.push((team, public, Value::Array(kept))), + Err(error) => { + team_model.push((team, public, positions)); + team_model.extend(rest); + self.team_model = team_model; + return Err(error); + } + } + } + self.team_model = team_model; + let mut publics: Vec = Vec::new(); + for (_, public, _) in &self.team_model { + if !publics.contains(public) { + publics.push(public.clone()); + } + } + self.team_public_model_names = publics; + self.patterns.remove(id); + for (_, router) in &mut self.team_patterns { + router.remove(id); + } + self.team_patterns + .retain(|(_, router)| !router.patterns.is_empty()); + Ok(()) + } + + /// Whether the id map holds `id` (`_has_model_id`). + pub fn has_id(&self, id: &str) -> bool { + self.id_map.contains_key(id) + } + + /// What the id map holds for `id`, as written (`model_id_to_deployment_index_map.get(id)`). + pub fn id_entry(&self, id: &str) -> Option<&Value> { + self.id_map.get(id) + } + + /// What the name index holds for `name`, as written. + pub fn named(&self, name: &str) -> Option<&Value> { + self.model_name.get(name) + } + + /// What the team index holds for `(team, public)`, as written. + pub fn team_named(&self, team: &str, public: &str) -> Option<&Value> { + self.team_model + .iter() + .find(|(t, p, _)| t == team && p == public) + .map(|(_, _, positions)| positions) + } + + /// `team_public_model_names_of`: the public names the team index holds for `team`. + pub fn team_public_names_of(&self, team: &str) -> Vec { + let mut names: Vec = Vec::new(); + for (t, public, _) in &self.team_model { + if t == team && !names.contains(public) { + names.push(public.clone()); + } + } + names + } + + /// The position the id map holds for `id`, when it is a position. + pub fn position_of(&self, id: &str) -> Option { + self.id_map + .get(id) + .and_then(Value::as_u64) + .and_then(|position| usize::try_from(position).ok()) + } +} + +impl PatternRouter { + /// `add_pattern`: a new regex joins in specificity order (a stable sort, most specific + /// first); a known one gains the deployment. + pub fn add(&mut self, pattern: &str, deployment: Value) { + let regex = pattern_to_regex(pattern); + if let Some((_, deployments)) = self.patterns.iter_mut().find(|(held, _)| *held == regex) { + deployments.push(deployment); + return; + } + self.patterns.push((regex, vec![deployment])); + self.patterns + .sort_by_key(|(regex, _)| std::cmp::Reverse(regex_specificity(regex))); + } + + /// `remove_deployment`: every deployment with this id goes, and so does a pattern left empty. + pub fn remove(&mut self, id: &str) { + for (_, deployments) in &mut self.patterns { + deployments.retain(|deployment| { + deployment + .get("model_info") + .and_then(|info| info.get("id")) + .and_then(Value::as_str) + != Some(id) + }); + } + self.patterns + .retain(|(_, deployments)| !deployments.is_empty()); + } + + /// `route`: the deployments of the first pattern `request` matches, each a copy whose + /// `litellm_params.model` takes the parts the wildcards matched; `filtered` keeps to the + /// patterns of those names. + pub fn route(&self, request: &str, filtered: Option<&[String]>) -> Option> { + let filtered: Option> = + filtered.map(|names| names.iter().map(|name| pattern_to_regex(name)).collect()); + for (regex, deployments) in &self.patterns { + if filtered + .as_ref() + .is_some_and(|names| !names.contains(regex)) + { + continue; + } + let Some(groups) = regex_match(regex, request) else { + continue; + }; + let mut routed = Vec::with_capacity(deployments.len()); + for deployment in deployments { + let mut copy = deployment.clone(); + let model = deployment + .get("litellm_params") + .and_then(|params| params.get("model")); + let Some(Value::String(model)) = model else { + // `deployment["litellm_params"]["model"]` raising ends the route with None. + return None; + }; + let named = deployment_model_name(&groups, request, model); + if model.contains('*') + && let Some(params) = + deployment.get("litellm_params").and_then(Value::as_object) + && let Some(info) = copy.get_mut("model_info").and_then(Value::as_object_mut) + { + for flag in cache_flags_left_to_the_catalog(params) { + info.remove(*flag); + } + } + if let Some(params) = copy + .get_mut("litellm_params") + .and_then(Value::as_object_mut) + { + params.insert("model".to_owned(), Value::String(named)); + } + routed.push(copy); + } + return Some(routed); + } + None + } + + /// `get_models_by_pattern`'s pick: the first (request, pattern) pair that matches, trying + /// `model` against every pattern, then `provider/model`. + pub fn models_by_pattern(&self, requests: &[String]) -> Option> { + for request in requests { + for (regex, deployments) in &self.patterns { + if let Some(groups) = regex_match(regex, request) { + return Some( + deployments + .iter() + .filter_map(|deployment| { + deployment + .get("litellm_params") + .and_then(|params| params.get("model")) + .and_then(Value::as_str) + .map(|model| deployment_model_name(&groups, request, model)) + }) + .collect(), + ); + } + } + } + None + } +} + +/// `_cache_flags_left_to_the_catalog`: the `model_info` flags a wildcard deployment does not speak +/// for on the models it matches. An azure_ai wildcard's `supports_prompt_cache_breakpoint` does. +/// `inferred_provider` names `azure_ai` only for a model carrying that prefix. +pub(crate) fn cache_flags_left_to_the_catalog( + params: &Map, +) -> &'static [&'static str] { + let provider = params + .get("custom_llm_provider") + .filter(|provider| truthy(provider)) + .and_then(Value::as_str) + .or_else(|| { + params + .get("model") + .and_then(Value::as_str) + .and_then(|model| model.split_once('/')) + .map(|(prefix, _)| prefix) + }); + if provider == Some("azure_ai") { + &["supports_prompt_caching"] + } else { + &[ + "supports_prompt_cache_breakpoint", + "supports_prompt_caching", + ] + } +} + +/// `re.escape(pattern).replace(r"\*", "(.*)")`. +pub fn pattern_to_regex(pattern: &str) -> String { + let mut regex = String::new(); + for c in pattern.chars() { + match c { + '*' => regex.push_str("(.*)"), + '(' | ')' | '[' | ']' | '{' | '}' | '?' | '+' | '-' | '|' | '^' | '$' | '\\' | '.' + | '&' | '~' | '#' | ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c' => { + regex.push('\\'); + regex.push(c); + } + _ => regex.push(c), + } + } + regex +} + +/// `PatternUtils.calculate_pattern_specificity`: the regex's length, then how much regex syntax +/// it has. +pub fn regex_specificity(regex: &str) -> (usize, usize) { + let complexity = regex + .chars() + .filter(|c| matches!(c, '*' | '+' | '?' | '\\' | '^' | '$' | '|' | '(' | ')')) + .count(); + (regex.chars().count(), complexity) +} + +/// A pattern regex read back as its literal runs between wildcards, or None when it holds regex +/// syntax `pattern_to_regex` never writes (a caller wrote it). +fn literal_runs(regex: &str) -> Option> { + let mut runs = vec![String::new()]; + let mut chars = regex.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' => runs.last_mut()?.push(chars.next()?), + '(' => { + let rest: String = chars.by_ref().take(3).collect(); + if rest != ".*)" { + return None; + } + runs.push(String::new()); + } + '.' | '*' | '+' | '?' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | ')' => return None, + other => runs.last_mut()?.push(other), + } + } + Some(runs) +} + +/// `re.match(regex, request)` for a pattern regex: anchored at the start only, each `(.*)` +/// greedy. The groups it captured, or None when it does not match. +pub fn regex_match(regex: &str, request: &str) -> Option> { + let runs = literal_runs(regex)?; + let rest = request.strip_prefix(runs[0].as_str())?; + let mut groups = Vec::new(); + greedy(&runs[1..], rest, &mut groups).then_some(groups) +} + +/// Match `runs` (each a literal that follows a group) against `rest`, longest group first. +fn greedy(runs: &[String], rest: &str, groups: &mut Vec) -> bool { + let Some((literal, later)) = runs.split_first() else { + return true; + }; + let mut ends: Vec = rest.char_indices().map(|(at, _)| at).collect(); + ends.push(rest.len()); + for &end in ends.iter().rev() { + let (group, after) = rest.split_at(end); + let Some(after) = after.strip_prefix(literal.as_str()) else { + continue; + }; + groups.push(group.to_owned()); + if greedy(later, after, groups) { + return true; + } + groups.pop(); + } + false +} + +/// `set_deployment_model_name`: the deployment's model with each wildcard replaced by the group +/// matched for it, the request itself when more groups matched than it has wildcards. +pub fn deployment_model_name(groups: &[String], request: &str, model: &str) -> String { + if !model.contains('*') { + return model.to_owned(); + } + if groups.len() > model.matches('*').count() { + return request.to_owned(); + } + let mut named = model.to_owned(); + for group in groups { + named = named.replacen('*', group, 1); + } + named +} + +fn unique(values: Vec) -> Vec { + let mut kept: Vec = Vec::with_capacity(values.len()); + for value in values { + if !kept.contains(&value) { + kept.push(value); + } + } + kept +} + +fn push_position(held: &mut Value, position: &Value) { + if let Value::Array(positions) = held { + positions.push(position.clone()); + } +} + +/// Python's truthiness for a JSON value. +pub(crate) fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64() != Some(0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(items) => !items.is_empty(), + } +} + +/// Python's `str()` for an id or team id. +pub(crate) fn python_str(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Bool(true) => "True".to_owned(), + Value::Bool(false) => "False".to_owned(), + Value::Null => "None".to_owned(), + other => other.to_string(), + } +} + +fn type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +/// `idx - 1 if idx > removal_idx else idx`: None keeps `held` as is. +fn shift(held: &Value, position: usize) -> Result, RemovalError> { + let removal = position as f64; + let number = match held { + Value::Bool(flag) => f64::from(u8::from(*flag)), + Value::Number(number) => number.as_f64().unwrap_or_default(), + other => { + return Err(RemovalError { + found: type_name(other), + }); + } + }; + if number <= removal { + return Ok(None); + } + Ok(Some(match held { + Value::Number(n) if n.is_i64() || n.is_u64() => Value::from(n.as_i64().unwrap_or(0) - 1), + _ => Value::from(number - 1.0), + })) +} + +/// `_without_indices`: the positions without `position`, the later ones shifted down. +fn without(held: &Value, position: usize) -> Result, RemovalError> { + let Value::Array(positions) = held else { + return Ok(Vec::new()); + }; + let removal = Value::from(position); + let mut kept = Vec::with_capacity(positions.len()); + for index in positions { + if *index == removal { + continue; + } + kept.push(shift(index, position)?.unwrap_or_else(|| index.clone())); + } + Ok(kept) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn entry(name: &str, id: &str, model: &str) -> Map { + serde_json::from_value(json!({ + "model_name": name, + "litellm_params": {"model": model}, + "model_info": {"id": id}, + })) + .unwrap() + } + + #[test] + fn removal_shifts_every_index_and_drops_emptied_names() { + let mut indexes = HostIndexes::default(); + indexes.added(&entry("gpt", "a", "openai/a"), 0, false); + indexes.added(&entry("gpt", "b", "openai/b"), 1, false); + indexes.added(&entry("claude", "c", "anthropic/c"), 2, false); + indexes.set_model_names(vec!["gpt".into(), "claude".into(), "broken".into()]); + indexes.removed("a", 0).unwrap(); + assert_eq!( + indexes.id_map, + json!({"b": 0, "c": 1}).as_object().unwrap().clone() + ); + assert_eq!(indexes.model_name["gpt"], json!([0])); + indexes.removed("c", 1).unwrap(); + assert!(!indexes.model_name.contains_key("claude")); + assert_eq!(indexes.model_names, vec![json!("gpt"), json!("broken")]); + assert_eq!(indexes.deployment_names.len(), 3); + } + + #[test] + fn a_pattern_routes_with_the_matched_parts() { + let mut router = PatternRouter::default(); + router.add( + "llmengine/*", + json!({"litellm_params": {"model": "openai/*"}, "model_info": {"id": "w"}}), + ); + router.add( + "*meta.llama3*", + json!({"litellm_params": {"model": "bedrock/meta.llama3*"}, "model_info": {"id": "m"}}), + ); + let routed = router.route("llmengine/foo", None).unwrap(); + assert_eq!(routed[0]["litellm_params"]["model"], "openai/foo"); + let routed = router.route("hello-world-meta.llama3-70b", None).unwrap(); + assert_eq!( + routed[0]["litellm_params"]["model"], + "hello-world-meta.llama3-70b" + ); + assert_eq!( + regex_match(r"a(.*)b(.*)", "axbyb").unwrap(), + vec!["xby", ""] + ); + router.remove("w"); + assert!(router.route("llmengine/foo", None).is_none()); + } + + #[rstest] + #[case::any_wildcard( + json!({"model": "openai/*"}), + json!({"id": "w", "keep": 1}), + )] + #[case::azure_ai_keeps_the_breakpoint_flag( + json!({"model": "azure_ai/*"}), + json!({"id": "w", "keep": 1, "supports_prompt_cache_breakpoint": true}), + )] + #[case::azure_ai_named_by_custom_llm_provider( + json!({"model": "*", "custom_llm_provider": "azure_ai"}), + json!({"id": "w", "keep": 1, "supports_prompt_cache_breakpoint": true}), + )] + #[case::a_fixed_model_speaks_for_itself( + json!({"model": "openai/gpt-4o"}), + json!({ + "id": "w", + "keep": 1, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + }), + )] + fn a_wildcard_leaves_the_cache_flags_to_the_catalog( + #[case] params: Value, + #[case] expected: Value, + ) { + let mut router = PatternRouter::default(); + router.add( + "any/*", + json!({ + "litellm_params": params, + "model_info": { + "id": "w", + "keep": 1, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + }, + }), + ); + let routed = router.route("any/gpt-4o", None).unwrap(); + assert_eq!(routed[0]["model_info"], expected); + } +} diff --git a/litellm-rust/crates/router-core/src/lib.rs b/litellm-rust/crates/router-core/src/lib.rs index 05d95d02c84..5f15b6b8df0 100644 --- a/litellm-rust/crates/router-core/src/lib.rs +++ b/litellm-rust/crates/router-core/src/lib.rs @@ -43,21 +43,706 @@ //! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the //! policy is exhausted. //! -//! # Execution -//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs -//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A -//! started attempt is never replayed on a different executor. +//! # Hosts: the only boundary +//! The core reaches outside itself only through [`Host`], and every attempt runs through +//! `Host::invoke`: the core picks the deployment, the host runs the call, the core records the +//! outcome. There is no second, "native" attempt path in the core. Which code runs an attempt is +//! the host's business: +//! - `PythonHost` (inside a Python process) builds the request for the picked deployment and calls +//! `litellm.`. It never calls back into a Python router. //! -//! # Hosts -//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select` -//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks, -//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all -//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost` -//! serves them from the existing Python code when the router runs inside Python; `NativeHost` -//! serves them in the Rust server, where an operation without a Rust implementation makes the -//! config fail at build time, never partway through a request. +//! An op or provider a host cannot run fails `Host::supports` when the router is built or +//! changed, never partway through a request. +//! +//! The core also builds its deployments through the host ([`build`]): for each configured entry +//! it finds the secret references and asks `secrets`, then `validate`, `provider` and `added`. +//! +//! Besides `invoke` and `supports`, a host provides `select` for strategies and plugins not yet +//! ported, `allow_fallback` for access and budget checks, `refuses_fallbacks` for a chain's dict +//! targets, `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that +//! holds all routing state under the same keys, encodings and TTLs the Python router uses. The +//! core writes cooldowns, allowed-fails counters and usage itself, through that cache. //! //! # Plugins //! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive) //! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and //! the current request), never the router itself. +//! +//! # Modules +//! - [`registry`]: deployments and their indices, `deployment_id`, `ChangeSet`. +//! - [`config`] / [`settings`]: static and dynamic config, and the frozen `Settings` view. +//! - [`cache`] / [`health`]: the shared routing cache, cooldowns and usage. +//! - [`select`]: resolve, filter chain, simple-shuffle. +//! - [`call`]: retries and fallbacks. +//! - [`eligibility`]: the endpoints a deployment refuses to serve. +//! - [`precall`]: the `enable_pre_call_checks` filter. +//! - [`model_info`]: model info merged from the cost map, asked of the host at query time. +//! - [`host`]: the `Host` boundary. + +pub mod access; +pub mod build; +pub mod cache; +pub mod call; +pub mod config; +pub mod eligibility; +pub mod error; +pub mod fallback; +pub mod group_info; +pub mod health; +pub mod health_routing; +pub mod host; +pub mod host_index; +pub mod lookup; +pub mod mock; +pub mod model_info; +pub mod outcome; +pub mod precall; +pub mod prerouting; +pub mod ptu; +pub mod registry; +pub mod resolve; +pub mod retry; +pub mod routing_groups; +pub mod routing_plugins; +pub mod select; +pub mod settings; +pub mod strategy; +pub mod weights; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use arc_swap::ArcSwap; +use parking_lot::Mutex; + +pub use crate::{ + cache::{InMemoryRouterCache, RouterCache}, + call::RouterResponse, + config::{Op, RouterConfig, RouterDynamicConfig, RoutingStrategy}, + error::{AttemptError, ConfigError, Error, FailureKind, Payload, Unsupported}, + health::{Clock, Health, Outcome, SystemClock}, + host::{ + Attempt, Blocked, BuildAsk, BuildPath, Built, Host, Output, Provided, RetryCause, + RouterEvent, RouterRequest, RoutingContext, SecretAt, SecretRef, SelectStage, + }, + outcome::{OutcomeCallbacks, OutcomeWrite}, + prerouting::{PreRoute, PreRouted}, + registry::{ChangeSet, Deployment, DeploymentSpec, Registry, RegistryVersion}, + select::{Randomness, SeededRandomness, ThreadRandomness}, + settings::Settings, +}; +use crate::{ + health::{GroupShape, routing_group_has_alternatives, team_model_has_alternatives}, + registry::DeploymentFilter, + select::SelectInputs, +}; + +/// The registry and settings one call reads, swapped atomically by `apply`. +pub(crate) struct RouterStateSnapshot { + pub registry: Arc, + pub settings: Arc, +} + +impl RouterStateSnapshot { + /// `_get_routing_context` for a pick of `name`: unless the request overrides the strategy, + /// a stale member map naming a group no longer configured raises `KeyError(group)`. + pub(crate) fn missing_routing_group( + &self, + request: &RouterRequest, + name: &str, + ) -> Result<(), Error> { + if strategy::request_override(request.routing_strategy.as_deref()).is_some() { + return Ok(()); + } + match self.registry.missing_member_group(name) { + Some(group) => Err(Error::MissingRoutingGroup { + group: group.to_owned(), + }), + None => Ok(()), + } + } +} + +pub(crate) struct Core { + host: Arc, + rng: Arc, + health: Health, + config: RouterConfig, + snapshot: ArcSwap, + writer: Mutex<()>, + retired: AtomicBool, +} + +impl Core { + /// Reject what this host cannot serve, before any request. + fn validate(&self, registry: &Registry, settings: &Settings) -> Result<(), ConfigError> { + let can_select = self.host.can_select(); + let strategies = std::iter::once(&settings.routing_strategy).chain( + registry + .routing_groups() + .iter() + .map(|group| &group.routing_strategy), + ); + for strategy in strategies { + if !strategy.is_native() && !can_select { + return Err(Unsupported::Strategy(strategy.clone()).into()); + } + } + if let Some(plugin) = settings.plugins.first() + && !can_select + { + return Err(Unsupported::Plugin(plugin.name.clone()).into()); + } + for deployment in registry.deployments(&DeploymentFilter::default()) { + self.validate_deployment(&deployment)?; + } + Ok(()) + } + + /// Reject a deployment whose provider this host cannot run a served op on. + fn validate_deployment(&self, deployment: &Deployment) -> Result<(), ConfigError> { + for op in &self.config.served_ops { + if !self.host.supports(op, deployment.provider()) { + return Err(Unsupported::Op { + op: op.clone(), + provider: deployment.provider().to_owned(), + } + .into()); + } + } + Ok(()) + } +} + +/// How a reported failure is recorded. +#[derive(Clone, Debug, Default)] +pub struct FailureReport { + /// Cool down for the deployment's own `cooldown_time` (then the provider's retry-after), + /// as the failure callback does, rather than the router default. + pub deployment_cooldown: bool, + /// The `cooldown_time` the failed call's own params carried (`model_info`, then + /// `litellm_params`), which wins over the provider's retry-after for a callback report. + pub cooldown_time: Option, + /// Count the failure only: its status is one no cooldown rule reads (an empty status). + pub count_only: bool, + /// The model group the request was for, whose fallbacks keep a lone deployment cooling. + pub requested_group: Option, + /// Cool down without counting the failure (`set_cooldown_deployments` alone). + pub uncounted: bool, + /// Also count the failed request against the deployment's per-minute requests + /// (`async_deployment_callback_on_failure`). + pub counts_request: bool, + /// The model the request's usage is keyed by (metadata's `deployment`, which a wildcard route + /// fills), when it is not the deployment's own. + pub usage_model: Option, + /// The failure callbacks of a logged failure; a report no callback makes keeps the default. + pub callbacks: OutcomeCallbacks, +} + +/// How a reported success is recorded. +#[derive(Clone, Debug, Default)] +pub struct SuccessReport { + /// See [`host::Output::tracks_usage`]. + pub tracks_usage: Option, + /// See [`FailureReport::usage_model`]. + pub usage_model: Option, + /// See [`FailureReport::callbacks`]. + pub callbacks: OutcomeCallbacks, +} + +/// Builds a [`Router`], with test seams. +pub struct RouterBuilder { + config: RouterConfig, + dynamic: RouterDynamicConfig, + host: Arc, + rng: Arc, + clock: Arc, +} + +impl RouterBuilder { + #[must_use] + pub fn randomness(self, rng: Arc) -> Self { + Self { rng, ..self } + } + + #[must_use] + pub fn clock(self, clock: Arc) -> Self { + Self { clock, ..self } + } + + pub fn build(self) -> Result { + let registry = Registry::build( + self.config.model_list.clone(), + self.dynamic.model_alias_map.clone(), + self.config.routing_groups.clone(), + )? + .with_member_groups(self.config.routing_group_members.clone()); + let settings = Settings::new(&self.config, Arc::new(self.dynamic)); + let core = Core { + health: Health::new(self.host.cache(), self.clock), + host: self.host, + rng: self.rng, + config: self.config, + snapshot: ArcSwap::from_pointee(RouterStateSnapshot { + registry: Arc::new(Registry::default()), + settings: Arc::new(settings.clone()), + }), + writer: Mutex::new(()), + retired: AtomicBool::new(false), + }; + core.validate(®istry, &settings)?; + core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: Arc::new(settings), + })); + Ok(Router { + core: Arc::new(core), + }) + } +} + +/// The router: the entry points `litellm.Router` delegates to. +#[derive(Clone)] +pub struct Router { + core: Arc, +} + +impl Router { + pub fn new( + config: RouterConfig, + dynamic: RouterDynamicConfig, + host: Arc, + ) -> Result { + Self::builder(config, dynamic, host).build() + } + + pub fn builder( + config: RouterConfig, + dynamic: RouterDynamicConfig, + host: Arc, + ) -> RouterBuilder { + RouterBuilder { + config, + dynamic, + host, + rng: Arc::new(ThreadRandomness), + clock: Arc::new(SystemClock), + } + } + + fn live(&self) -> Result<(), Error> { + if self.core.retired.load(Ordering::Acquire) { + return Err(Error::Retired); + } + Ok(()) + } + + /// Select, run the attempt, and retry or fall back until it succeeds or the policy is + /// exhausted. + pub async fn call(&self, op: Op, request: RouterRequest) -> Result { + self.live()?; + self.core.call(op, Arc::new(request)).await + } + + /// Resolve, filter and pick a deployment without calling it. + pub async fn select(&self, op: &Op, request: &RouterRequest) -> Result, Error> { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + routing_plugins::refuse_sync_pick( + snapshot.settings.routing_plugins, + &snapshot.settings.routing_strategy, + request, + )?; + let routed = self + .core + .pre_routed(PreRoute { + op, + request, + name: &request.model, + fallback_depth: 0, + retry: 0, + }) + .await?; + let name = match routed { + PreRouted::Model(name) => name, + PreRouted::Deployment(deployment) => return Ok(deployment), + }; + snapshot.missing_routing_group(request, &name)?; + let candidates = self + .core + .candidates(&snapshot, op, request, &name, &SelectInputs::default()) + .await?; + self.core.pick(&snapshot, op, request, &candidates).await + } + + /// The deployments a pick for `request` would choose among: resolved, team- and + /// cooldown-filtered, through the plugins and the lowest `order`. + pub async fn healthy( + &self, + op: &Op, + request: &RouterRequest, + ) -> Result { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + let candidates = self + .core + .candidates( + &snapshot, + op, + request, + &request.model, + &SelectInputs::default(), + ) + .await?; + // `_core_async_get_healthy_deployments` raises when nothing is left. + if candidates.healthy.is_empty() { + return Err(Error::NoHealthyDeployments { + model: candidates.failed_model(&snapshot.registry), + }); + } + Ok(candidates) + } + + pub fn registry(&self) -> Arc { + self.core.snapshot.load().registry.clone() + } + + pub fn settings(&self) -> Arc { + self.core.snapshot.load().settings.clone() + } + + pub fn health(&self) -> &Health { + &self.core.health + } + + /// Record an outcome the router did not see itself, such as a health check. Returns whether + /// the deployment was cooled down. + pub async fn report(&self, id: &str, outcome: Outcome) -> Result { + match outcome { + Outcome::Success { total_tokens } => self + .report_success(id, total_tokens as f64, 1.0, SuccessReport::default()) + .await + .map(|_| false), + Outcome::Failure(failure) => self + .report_failure(id, &failure, FailureReport::default()) + .await + .map(|cooldown| cooldown.is_some()), + } + } + + /// A success a logging callback saw (`deployment_callback_on_success`): counted for a + /// configured deployment, with its tokens and requests when it tracks usage. Returns the + /// success counter's key, or `None` for an unknown deployment. + pub async fn report_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + report: SuccessReport, + ) -> Result, Error> { + report + .callbacks + .record( + OutcomeWrite::Success, + self.reported_success(id, total_tokens, requests, &report), + ) + .await + } + + async fn reported_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + report: &SuccessReport, + ) -> Result, Error> { + let snapshot = self.core.snapshot.load_full(); + let Some(deployment) = snapshot.registry.deployment(id) else { + return Ok(None); + }; + let key = self.core.health.count_success(id).await?; + if report + .tracks_usage + .unwrap_or_else(|| deployment.tracks_usage()) + { + self.core + .health + .add_usage( + &deployment, + report.usage_model.as_deref(), + total_tokens, + requests, + ) + .await?; + } + Ok(Some(key)) + } + + /// `sync_deployment_callback_on_success`: count a success against `id`, configured or not, + /// and return the counter's key. + pub async fn count_success( + &self, + id: &str, + callbacks: OutcomeCallbacks, + ) -> Result, Error> { + callbacks + .record(OutcomeWrite::Success, async { + self.core.health.count_success(id).await.map(Some) + }) + .await + } + + /// A failure the router did not see itself: counted against `id` (known or not), then + /// cooled down by the rules (`record_failure`, `deployment_callback_on_failure`). Returns the + /// cooldown applied, if any. + pub async fn report_failure( + &self, + id: &str, + failure: &AttemptError, + report: FailureReport, + ) -> Result, Error> { + report + .callbacks + .record( + OutcomeWrite::Failure, + self.reported_failure(id, failure, &report), + ) + .await + } + + async fn reported_failure( + &self, + id: &str, + failure: &AttemptError, + report: &FailureReport, + ) -> Result, Error> { + if !report.uncounted && !failure.outside_deployment { + self.core.health.count_failure(id).await?; + } + let snapshot = self.core.snapshot.load_full(); + let Some(deployment) = snapshot.registry.deployment(id) else { + return Ok(None); + }; + if report.counts_request { + self.core + .health + .count_failed_request(&deployment, report.usage_model.as_deref()) + .await?; + } + if failure.outside_deployment { + return Ok(None); + } + // An empty status is no cooldown (`_is_cooldown_required`), unless the deployment's + // own policy names the error. + if report.count_only + && crate::health::deployment_allowed_fails( + deployment.model_info.allowed_fails_policy.as_ref(), + failure, + ) + .is_none() + { + return Ok(None); + } + let cooldown = if report.deployment_cooldown { + report + .cooldown_time + .filter(|seconds| *seconds >= 0.0) + .and_then(|seconds| std::time::Duration::try_from_secs_f64(seconds).ok()) + .or(failure.retry_after) + .unwrap_or(snapshot.settings.cooldown_time) + } else { + snapshot.settings.cooldown_time + }; + let shape = GroupShape { + single_deployment: snapshot + .registry + .group(&deployment.model_name) + .is_some_and(|group| group.len() == 1) + && !report + .requested_group + .as_deref() + .is_some_and(|group| routing_group_has_alternatives(&snapshot.registry, group)) + && !team_model_has_alternatives(&snapshot.registry, &deployment), + has_fallback_group: snapshot + .settings + .dynamic + .fallback_targets(&[report + .requested_group + .as_deref() + .unwrap_or(&deployment.model_name)]) + .is_some(), + }; + self.core + .health + .cool_down_after(&snapshot.settings, &deployment, shape, failure, cooldown) + .await + } + + /// Apply one change set atomically and return the new registry version. + pub fn apply(&self, changes: ChangeSet) -> Result { + let _writer = self.core.writer.lock(); + let current = self.core.snapshot.load_full(); + let aliases = changes + .settings + .as_ref() + .map(|dynamic| dynamic.model_alias_map.clone()); + let registry = current.registry.apply(&changes, aliases)?; + let settings = match changes.settings { + Some(dynamic) => Settings::new(&self.core.config, Arc::new(dynamic)), + None => Settings::clone(¤t.settings), + }; + self.core.validate(®istry, &settings)?; + let version = registry.version(); + self.core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: Arc::new(settings), + })); + Ok(version) + } + + /// `get_router_model_info` for one deployment, its cost-map reads asked of the host now + /// ([`model_info::router_model_info`]). + pub async fn router_model_info( + &self, + query: &model_info::RouterModelInfoQuery, + ) -> Result, Error> { + model_info::router_model_info(self.core.host.as_ref(), query).await + } + + /// `_set_model_group_info` for the deployments `query` names ([`group_info::model_group_info`]). + pub async fn model_group_info( + &self, + query: &group_info::GroupInfoQuery, + ) -> Result, Error> { + group_info::model_group_info(self.core.host.as_ref(), query).await + } + + /// `get_configured_token_limits` for deployment `id` ([`group_info::configured_token_limits`]). + pub async fn configured_token_limits( + &self, + id: serde_json::Value, + configured: &serde_json::Map, + ) -> Result<(Option, Option), Error> { + group_info::configured_token_limits(self.core.host.as_ref(), id, configured).await + } + + /// `get_model_listing_info` over a listed name's deployments ([`group_info::model_listing_info`]). + pub async fn model_listing_info( + &self, + deployments: &[serde_json::Value], + ) -> Result { + group_info::model_listing_info(self.core.host.as_ref(), deployments).await + } + + /// `get_deployment_model_info` for deployment `model_id` serving `model_name` + /// ([`model_info::deployment_model_info`]). + pub async fn deployment_model_info( + &self, + model_id: &str, + model_name: &serde_json::Value, + ) -> Result>, Error> { + model_info::deployment_model_info(self.core.host.as_ref(), model_id, model_name).await + } + + /// Build `entries` through the host ([`build::build`]), installing each deployment it + /// accepts as soon as it does, as `install` says: a replace empties the registry first (and + /// takes `set_model_list`'s `model_names` once every entry is built), an append or upsert + /// adds at the end. When a step fails, what was installed before it stays, as the Python + /// router keeps the deployments it added before the one that raised. + /// + /// The build installs into a registry of its own, indexed in place, and serves it once it + /// ends (or fails), so n deployments build in O(n). A write that lands while it runs (a step + /// answer that applies a changeset) is kept: what the build installed is added over it. + pub async fn build( + &self, + path: BuildPath, + entries: Vec>, + resolve_secrets: bool, + install: build::Install, + ) -> Result, Error> { + let model_names: Vec = entries + .iter() + .map(|entry| match entry.get("model_name") { + Some(serde_json::Value::String(name)) => name.clone(), + None | Some(serde_json::Value::Null) => "None".to_owned(), + Some(other) => other.to_string(), + }) + .collect(); + let replace = install == build::Install::Replace; + let base = self.core.snapshot.load_full().registry.clone(); + let mut working = if replace { + base.emptied() + } else { + Registry::clone(&base) + }; + let mut installed: Vec = Vec::new(); + let add = path == BuildPath::Add; + let core = &self.core; + let built = build::build( + core.host.as_ref(), + path, + entries, + resolve_secrets, + |deployment| { + working.push_built(deployment.spec.clone(), add)?; + installed.push(deployment.spec.clone()); + core.validate_deployment(working.last().expect("just installed")) + .map_err(Error::from) + }, + ) + .await; + if built.is_ok() && replace && path == BuildPath::Create { + working.set_model_names(model_names); + } + self.serve_built(&base, working, installed, add)?; + built + } + + /// Serve the registry a build made from `base`. When another write replaced `base` while + /// the build ran, the build's deployments are added over that write instead. + fn serve_built( + &self, + base: &Arc, + built: Registry, + installed: Vec, + add: bool, + ) -> Result<(), ConfigError> { + let _writer = self.core.writer.lock(); + let current = self.core.snapshot.load_full(); + let registry = if Arc::ptr_eq(¤t.registry, base) { + built + } else { + let mut next = Registry::clone(¤t.registry); + for spec in installed { + next.push_built(spec, add)?; + } + next + }; + self.core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: current.settings.clone(), + })); + Ok(()) + } + + /// Keep the providers `from`'s host resolved for the deployments this router serves too (a + /// router rebuilt over the same deployments for new static settings). + pub fn adopt_providers(&self, from: &Self) { + let _writer = self.core.writer.lock(); + let current = self.core.snapshot.load_full(); + let registry = current + .registry + .with_providers_of(&from.core.snapshot.load().registry); + self.core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: current.settings.clone(), + })); + } + + /// Stop serving: every later call and selection fails with [`Error::Retired`]. + pub fn retire(&self) { + self.core.retired.store(true, Ordering::Release); + } +} diff --git a/litellm-rust/crates/router-core/src/lookup.rs b/litellm-rust/crates/router-core/src/lookup.rs new file mode 100644 index 00000000000..39c235d6f7b --- /dev/null +++ b/litellm-rust/crates/router-core/src/lookup.rs @@ -0,0 +1,883 @@ +//! The registry lookups the host serves as `registry()` queries, answered over the deployments and +//! the host indexes ([`crate::host_index::HostIndexes`]) the way the Python router answers them. +//! +//! A lookup answers with [`Row`]s: positions in the deployment list (each with how the row renames +//! the deployment), wildcard routes, or a position the Python list could not be read at. The host +//! turns a position into the entry object it stores, so identity and in-place reads stay its own; +//! a [`Row::Raw`] is handed back for the host to index its list by, which raises exactly where the +//! Python router's own read raised (an index a caller wrote junk into). + +use std::sync::Arc; + +use serde_json::{Map, Value, json}; + +use crate::{ + host_index::{HostIndexes, python_str, truthy}, + registry::{Deployment, Registry}, +}; + +/// One row of a lookup's answer. +#[derive(Clone, Debug, PartialEq)] +pub enum Row { + /// The deployment at `position`, under `name` when the row renames it (an alias or a routing + /// group), as a routing group's member `member` of `group` when `group` is set (its row drops + /// the member's `access_groups` and takes the group's priority as its `order`). + At { + position: usize, + name: Option, + group: Option<(String, String)>, + }, + /// A wildcard route: a copy of a pattern router's deployment, under `name`. + Pattern { entry: Value, name: String }, + /// A position the list cannot be read at: the host indexes its list by it, and raises. + Raw(Value), + /// A position container that is not a list: the host iterates it as its list's positions. + RawPositions(Value), +} + +impl Row { + /// The row as the host reads it. + pub fn wire(&self) -> Value { + match self { + Self::At { + position, + name, + group, + } => { + let mut row = Map::new(); + row.insert("at".to_owned(), json!(position)); + if let Some(name) = name { + row.insert("name".to_owned(), json!(name)); + } + if let Some((group, member)) = group { + row.insert("group".to_owned(), json!(group)); + row.insert("member".to_owned(), json!(member)); + } + Value::Object(row) + } + Self::Pattern { entry, name } => json!({"pattern": entry, "name": name}), + Self::Raw(value) => json!({"raw": value}), + Self::RawPositions(value) => json!({"raw_positions": value}), + } + } +} + +/// An alias `model_list` lists: its name and the group it targets (`hidden` aliases already left +/// out by the host when the whole list is asked for). +#[derive(Clone, Debug, PartialEq, serde::Deserialize)] +pub struct AliasItem { + pub alias: String, + pub target: String, +} + +/// A routing group callable by its name (the host leaves out one a deployment or an alias +/// shadows): its name and member model groups. +#[derive(Clone, Debug, PartialEq, serde::Deserialize)] +pub struct CallableGroup { + pub name: String, + pub members: Vec, +} + +/// What `get_model_list` composes beyond the deployments: the aliases and callable routing groups +/// it lists, and the wildcard requests a name no deployment serves is routed by. +#[derive(Clone, Debug, Default, PartialEq, serde::Deserialize)] +#[serde(default)] +pub struct ListScope { + pub aliases: Vec, + pub groups: Vec, + /// `get_deployments_by_pattern`'s requests: the name, then the name under its provider. + pub wildcard: Vec, + /// Only one part of the list: `"aliases"` (`_core_get_model_list_from_model_alias`) or + /// `"groups"` (`_core_get_model_list_from_routing_groups`). + pub only: Option, +} + +/// A row read for a lookup's filters: the deployment's fields as the row shows them. +struct View<'a> { + name: &'a str, + info: &'a Map, + params: &'a Map, + strips_access_groups: bool, +} + +/// The lookups over one registry version. +pub struct Lookup<'a> { + deployments: &'a [Arc], + host: &'a HostIndexes, +} + +impl<'a> Lookup<'a> { + pub fn new(registry: &'a Registry) -> Self { + Self { + deployments: registry.deployments_from(0), + host: registry.host_indexes(), + } + } + + /// `list[value]` as Python reads it: an int (or bool) in range, negative from the end. + fn position(&self, value: &Value) -> Option { + let len = i64::try_from(self.deployments.len()).ok()?; + let index = match value { + Value::Bool(flag) => i64::from(*flag), + Value::Number(number) => number.as_i64()?, + _ => return None, + }; + let index = if index < 0 { len + index } else { index }; + (0..len) + .contains(&index) + .then(|| usize::try_from(index).ok()) + .flatten() + } + + /// Every position `held` (what an index holds) names, read before any is filtered, as the + /// Python router reads them into a tuple first: `Err` with the row that raises. + fn positions(&self, held: &Value) -> Result, Row> { + let items = match held { + Value::Array(items) => items, + // Iterating an empty string or dict reads no position; anything else raises. + Value::String(text) if text.is_empty() => return Ok(Vec::new()), + Value::Object(map) if map.is_empty() => return Ok(Vec::new()), + _ => return Err(Row::RawPositions(held.clone())), + }; + items + .iter() + .map(|item| self.position(item).ok_or_else(|| Row::Raw(item.clone()))) + .collect() + } + + fn view(&self, position: usize) -> View<'a> { + let deployment = &self.deployments[position]; + View { + name: &deployment.model_name, + info: &deployment.model_info.raw, + params: &deployment.raw_params, + strips_access_groups: false, + } + } + + fn row_view<'r>(&self, row: &'r Row) -> Option> + where + 'a: 'r, + { + match row { + Row::At { + position, + name, + group, + } => { + let base = self.view(*position); + Some(View { + name: name.as_deref().unwrap_or(base.name), + strips_access_groups: group.is_some(), + ..base + }) + } + Row::Pattern { entry, name } => Some(View { + name, + info: entry + .get("model_info") + .and_then(Value::as_object) + .unwrap_or(empty()), + params: entry + .get("litellm_params") + .and_then(Value::as_object) + .unwrap_or(empty()), + strips_access_groups: false, + }), + Row::Raw(_) | Row::RawPositions(_) => None, + } + } + + // -- deployments by name ------------------------------------------------------------------- + + /// `_core_get_all_deployments`: the team's deployments under public name `model_name`, else + /// the group's, else (for a team) every deployment the name includes; each under `rename`. + pub fn all(&self, model_name: &str, team_id: Option<&str>, rename: Option<&str>) -> Vec { + let rows = |positions: Vec| -> Vec { + positions + .into_iter() + .filter(|position| should_include(model_name, &self.view(*position), team_id)) + .map(|position| Row::At { + position, + name: rename.map(str::to_owned), + group: None, + }) + .collect() + }; + if let Some(team) = team_id + && let Some(held) = self.host.team_named(team, model_name) + { + match self.positions(held) { + Ok(positions) => { + let found = rows(positions); + if !found.is_empty() { + return found; + } + } + Err(raw) => return vec![raw], + } + } + if let Some(held) = self.host.named(model_name) { + return match self.positions(held) { + Ok(positions) => rows(positions), + Err(raw) => vec![raw], + }; + } + if team_id.is_some() { + return rows((0..self.deployments.len()).collect()); + } + Vec::new() + } + + /// `_team_deployments_across_teams`: every team's deployments under public name `model`, and + /// how many distinct teams own them. + pub fn across_teams(&self, model: &str) -> (Vec, usize) { + let mut rows = Vec::new(); + for (_, public, held) in &self.host.team_model { + if public != model { + continue; + } + match self.positions(held) { + Ok(positions) => rows.extend(positions.into_iter().map(|position| Row::At { + position, + name: None, + group: None, + })), + Err(raw) => return (vec![raw], 0), + } + } + let mut teams: Vec<&Value> = Vec::new(); + for row in &rows { + if let Some(view) = self.row_view(row) + && let Some(team) = view.info.get("team_id").filter(|team| !team.is_null()) + && !teams.contains(&team) + { + teams.push(team); + } + } + let count = teams.len(); + (rows, count) + } + + /// `_core_config_deployments`: the deployments a config declared (no `db_model: true`). + pub fn config(&self) -> Vec { + (0..self.deployments.len()) + .filter(|position| { + self.view(*position).info.get("db_model") != Some(&Value::Bool(true)) + }) + .map(|position| Row::At { + position, + name: None, + group: None, + }) + .collect() + } + + /// `_core_get_model_ids`: the ids of the group's deployments, or (no group) of the id map's + /// entries, without team deployments when `exclude_team_models`. `Err` with the row that raises. + pub fn model_ids( + &self, + model_name: Option<&str>, + exclude_team_models: bool, + ) -> Result, Row> { + let kept = |view: &View<'_>| { + view.info.contains_key("id") + && !(exclude_team_models && view.info.get("team_id").is_some_and(truthy)) + }; + if let Some(name) = model_name { + let Some(held) = self.host.named(name) else { + return Ok(Vec::new()); + }; + return Ok(self + .positions(held)? + .into_iter() + .map(|position| self.view(position)) + .filter(kept) + .map(|view| python_str(&view.info["id"])) + .collect()); + } + let mut ids = Vec::new(); + for (id, held) in &self.host.id_map { + let position = self.position(held).ok_or_else(|| Row::Raw(held.clone()))?; + if kept(&self.view(position)) { + ids.push(id.clone()); + } + } + Ok(ids) + } + + // -- the model list -------------------------------------------------------------------------- + + /// `_core_get_model_list`: the name's deployments, its alias rows and routing group rows, its + /// wildcard routes when nothing else serves it, or (no name) every alias and group row and + /// then every deployment. + pub fn model_list( + &self, + model_name: Option<&str>, + team_id: Option<&str>, + scope: &ListScope, + ) -> Vec { + let only = scope.only.as_deref(); + let mut rows = Vec::new(); + if let Some(name) = model_name + && only.is_none() + { + rows.extend(self.all(name, team_id, None)); + } + let aliases: &[AliasItem] = if only == Some("groups") { + &[] + } else { + &scope.aliases + }; + for alias in aliases { + match scope.groups.iter().find(|group| group.name == alias.target) { + Some(group) => { + rows.extend(self.group_rows(group).into_iter().map(|row| match row { + Row::At { + position, group, .. + } => Row::At { + position, + name: Some(alias.alias.clone()), + group, + }, + other => other, + })) + } + None => rows.extend(self.all(&alias.target, None, Some(&alias.alias))), + } + } + match (model_name, only) { + (_, Some("aliases")) => {} + (Some(name), _) => { + if let Some(group) = scope.groups.iter().find(|group| group.name == name) { + rows.extend(self.group_rows(group)); + } + } + (None, _) => { + for group in &scope.groups { + rows.extend(self.group_rows(group)); + } + } + } + if only.is_some() { + return rows; + } + if let Some(name) = model_name + && rows.is_empty() + { + rows.extend(self.wildcard(name, team_id, &scope.wildcard)); + } + if model_name.is_none() { + rows.extend((0..self.deployments.len()).map(|position| Row::At { + position, + name: None, + group: None, + })); + } + rows + } + + /// The whole list's rows, or the first row the host's read of the list raises at. + fn readable(&self, rows: Vec) -> Result, Row> { + match rows + .iter() + .find(|row| matches!(row, Row::Raw(_) | Row::RawPositions(_))) + { + Some(raw) => Err(raw.clone()), + None => Ok(rows), + } + } + + /// A callable routing group's rows: each member's deployments under the group's name. + fn group_rows(&self, group: &CallableGroup) -> Vec { + let mut rows = Vec::new(); + for member in &group.members { + for row in self.all(member, None, Some(&group.name)) { + rows.push(match row { + Row::At { position, name, .. } => Row::At { + position, + name, + group: Some((group.name.clone(), member.clone())), + }, + other => other, + }); + } + } + rows + } + + /// `_get_wildcard_deployments`: the global router's routes for the requests, then the team's. + fn wildcard(&self, name: &str, team_id: Option<&str>, requests: &[String]) -> Vec { + let route = |router: &crate::host_index::PatternRouter| -> Vec { + requests + .iter() + .find_map(|request| { + router + .route(request, None) + .filter(|routed| !routed.is_empty()) + }) + .unwrap_or_default() + }; + let mut routed = route(&self.host.patterns); + if let Some(team) = team_id + && let Some((_, router)) = self.host.team_patterns.iter().find(|(t, _)| t == team) + { + routed.extend(route(router)); + } + routed + .into_iter() + .map(|entry| Row::Pattern { + entry, + name: name.to_owned(), + }) + .collect() + } + + /// The deployments `team_id`'s own wildcard pattern routes `name` to + /// (`_core_team_pattern_routers`, keyed by `team_public_model_name`). + pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec> { + let Some((_, router)) = self.host.team_patterns.iter().find(|(t, _)| t == team_id) else { + return Vec::new(); + }; + router + .route(name, None) + .unwrap_or_default() + .iter() + .filter_map(|entry| entry.get("model_info")?.get("id").map(python_str)) + .filter_map(|id| self.deployments.iter().find(|d| d.id == id).cloned()) + .collect() + } + + /// `_core_get_model_names`: per row of the whole list, a team deployment's public name for + /// its own team, else the row's name. + pub fn model_names( + &self, + team_id: Option<&str>, + scope: &ListScope, + ) -> Result, Row> { + let rows = self.readable(self.model_list(None, None, scope))?; + let mut names = Vec::new(); + for row in &rows { + let Some(view) = self.row_view(row) else { + continue; + }; + match view.info.get("team_id").filter(|team| truthy(team)) { + Some(team) => { + if team_id.is_some_and(|id| team == &Value::String(id.to_owned())) + && let Some(public) = view + .info + .get("team_public_model_name") + .filter(|public| truthy(public)) + { + names.push(python_str(public)); + } + } + None => names.push(view.name.to_owned()), + } + } + Ok(names) + } + + /// `_core_get_model_access_groups`: access group to the names of the rows that declare it, + /// in the order they appear. + pub fn access_groups( + &self, + model_name: Option<&str>, + access_group: Option<&str>, + team_id: Option<&str>, + scope: &ListScope, + ) -> Result)>, Row> { + let mut groups: Vec<(String, Vec)> = Vec::new(); + for row in self.readable(self.model_list(model_name, team_id, scope))? { + let Some(view) = self.row_view(&row) else { + continue; + }; + if view.info.is_empty() || view.strips_access_groups { + continue; + } + // `access_groups or []`, iterated: a list's items, a string's characters. + let declared: Vec = match view.info.get("access_groups") { + Some(Value::Array(items)) => items.iter().map(python_str).collect(), + Some(Value::String(text)) => text.chars().map(String::from).collect(), + _ => continue, + }; + for group in declared { + if access_group.is_some_and(|wanted| wanted != group) { + continue; + } + match groups.iter_mut().find(|(held, _)| *held == group) { + Some((_, names)) => names.push(view.name.to_owned()), + None => groups.push((group, vec![view.name.to_owned()])), + } + } + } + Ok(groups) + } + + /// `_core_resolve_model_name_from_model_id`'s scan of the whole list: whether it lists any + /// row, and the name of the first row whose `litellm_params.model` is `model_id`, or ends in + /// `/model_id` or `:model_id`. + pub fn name_serving( + &self, + model_id: &str, + scope: &ListScope, + ) -> Result<(bool, Option), Row> { + let rows = self.readable(self.model_list(None, None, scope))?; + let (slash, colon) = (format!("/{model_id}"), format!(":{model_id}")); + let name = rows.iter().find_map(|row| { + let view = self.row_view(row)?; + let model = view.params.get("model").and_then(Value::as_str)?; + (model == model_id || model.ends_with(&slash) || model.ends_with(&colon)) + .then_some(view.name) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + }); + Ok((!rows.is_empty(), name)) + } + + // -- teams and credentials ------------------------------------------------------------------- + + /// `_core_team_model_has_alternatives`: whether the team's public name of deployment `id` has + /// another unpaused deployment of that team. + pub fn team_model_has_alternatives(&self, id: &str) -> Result { + let Some(held) = self.host.id_entry(id) else { + return Ok(false); + }; + let position = self.position(held).ok_or_else(|| Row::Raw(held.clone()))?; + let info = self.view(position).info; + let (Some(Value::String(team)), Some(Value::String(public))) = + (info.get("team_id"), info.get("team_public_model_name")) + else { + return Ok(false); + }; + let Some(siblings) = self.host.team_named(team, public) else { + return Ok(false); + }; + let unpaused = self + .positions(siblings)? + .into_iter() + .filter(|position| self.view(*position).info.get("blocked") != Some(&Value::Bool(true))) + .count(); + Ok(unpaused > 1) + } + + /// `_get_model_group_deployment_usable_by_team`: the first deployment of the group the team + /// may use. + pub fn group_deployment_usable_by_team( + &self, + model_group: &str, + team_id: Option<&str>, + ) -> Result, Row> { + // `.get(name) or ()`: nothing held, or a falsy value, reads no position. + let Some(held) = self.host.named(model_group).filter(|held| truthy(held)) else { + return Ok(None); + }; + let Value::Array(items) = held else { + return Err(Row::RawPositions(held.clone())); + }; + for item in items { + let position = self.position(item).ok_or_else(|| Row::Raw(item.clone()))?; + if usable_by_team(&self.view(position), team_id) { + return Ok(Some(position)); + } + } + Ok(None) + } + + /// `_get_team_public_name_deployment`: the first deployment of the team's public name. + pub fn team_public_name_deployment( + &self, + model_id: &str, + team_id: &str, + ) -> Result, Row> { + match self.host.team_named(team_id, model_id) { + Some(Value::Array(items)) => match items.first() { + Some(item) => self + .position(item) + .map(Some) + .ok_or_else(|| Row::Raw(item.clone())), + None => Ok(None), + }, + Some(held) if truthy(held) => Err(Row::RawPositions(held.clone())), + Some(_) | None => Ok(None), + } + } + + // -- routable deployments -------------------------------------------------------------------- + + /// `_routable_deployments`: the routable group's deployments for the team (those it may use + /// when any), without paused ones. + pub fn routable(&self, model_group: &str, team_id: Option<&str>) -> Vec { + let named = self.all(model_group, team_id, None); + if named + .iter() + .any(|row| matches!(row, Row::Raw(_) | Row::RawPositions(_))) + { + return named; + } + let usable: Vec<&Row> = named + .iter() + .filter(|row| { + self.row_view(row) + .is_some_and(|view| usable_by_team(&view, team_id)) + }) + .collect(); + let pool: Vec<&Row> = if usable.is_empty() { + named.iter().collect() + } else { + usable + }; + pool.into_iter() + .filter(|row| { + self.row_view(row) + .is_some_and(|view| view.info.get("blocked") != Some(&Value::Bool(true))) + }) + .cloned() + .collect() + } + + /// `_core_get_candidate_model_ids_for_route`'s ids of `rows`, in first-seen order. + pub fn ids_of(&self, rows: &[Row]) -> Result, Row> { + let mut ids: Vec = Vec::new(); + for row in rows { + let view = match row { + Row::Raw(_) | Row::RawPositions(_) => return Err(row.clone()), + _ => self.row_view(row).expect("a readable row"), + }; + if let Some(id) = view.info.get("id").filter(|id| !id.is_null()) { + let id = python_str(id); + if !ids.contains(&id) { + ids.push(id); + } + } + } + Ok(ids) + } + + /// `_core_get_candidate_model_ids_for_route` once the alias resolved `resolved`: the + /// members' ids when it names a callable routing group (`members`), else, for a name that is + /// no model group, the team's deployments under it or its wildcard routes, else the group's. + pub fn candidate_ids( + &self, + resolved: &str, + team_id: Option<&str>, + members: Option<&[String]>, + requests: &[String], + ) -> Result, Row> { + if let Some(members) = members { + let mut ids: Vec = Vec::new(); + for member in members { + for id in self.ids_of(&self.all(member, team_id, None))? { + if !ids.contains(&id) { + ids.push(id); + } + } + } + return Ok(ids); + } + let named = Value::String(resolved.to_owned()); + if !self.host.model_names.contains(&named) { + if team_id.is_some() { + let team_rows = self.all(resolved, team_id, None); + if !team_rows.is_empty() { + return self.ids_of(&team_rows); + } + } + let wildcard = self.wildcard_route(resolved, team_id, requests); + if !wildcard.is_empty() { + return self.ids_of(&wildcard); + } + } + self.ids_of(&self.all(resolved, team_id, None)) + } + + /// `_core_wildcard_route_deployments`: the team router's routes for `model`, else the global + /// router's. + pub fn wildcard_route( + &self, + model: &str, + team_id: Option<&str>, + requests: &[String], + ) -> Vec { + let route = |router: &crate::host_index::PatternRouter| -> Vec { + requests + .iter() + .find_map(|request| { + router + .route(request, None) + .filter(|routed| !routed.is_empty()) + }) + .unwrap_or_default() + }; + let team_owned = team_id + .and_then(|team| self.host.team_patterns.iter().find(|(t, _)| t == team)) + .map(|(_, router)| route(router)) + .unwrap_or_default(); + let routed = if team_owned.is_empty() { + route(&self.host.patterns) + } else { + team_owned + }; + routed + .into_iter() + .map(|entry| Row::Pattern { + entry, + name: model.to_owned(), + }) + .collect() + } +} + +/// `_core_should_include_deployment`: the team's deployment under its public name, or a +/// deployment of the group the team may use. +fn should_include(model_name: &str, view: &View<'_>, team_id: Option<&str>) -> bool { + let team_value = team_id.map(|team| Value::String(team.to_owned())); + let model_team = view.info.get("team_id"); + if let Some(team) = &team_value + && model_team == Some(team) + && view.info.get("team_public_model_name") == Some(&Value::String(model_name.to_owned())) + { + return true; + } + if view.name == model_name { + return team_value.is_none() + || model_team.is_none_or(Value::is_null) + || model_team == team_value.as_ref(); + } + false +} + +/// `deployment_usable_by_team`: a deployment no team owns, or one the team owns. +fn usable_by_team(view: &View<'_>, team_id: Option<&str>) -> bool { + match view.info.get("team_id").filter(|team| !team.is_null()) { + None => true, + Some(owner) => team_id.is_some_and(|team| owner == &Value::String(team.to_owned())), + } +} + +fn empty() -> &'static Map { + static EMPTY: std::sync::OnceLock> = std::sync::OnceLock::new(); + EMPTY.get_or_init(Map::new) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn registry(entries: Value) -> Registry { + let mut registry = Registry::default(); + for entry in entries.as_array().unwrap() { + let spec: DeploymentSpec = serde_json::from_value(entry.clone()).unwrap(); + registry.push_built(spec, false).unwrap(); + } + registry + } + + #[test] + fn a_teams_wildcard_routes_only_the_names_its_public_pattern_matches() { + let registry = registry(json!([ + {"model_name": "t1-w", "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "tw1", "team_id": "T1", "team_public_model_name": "tw-*"}}, + {"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "gp"}}, + ])); + let lookup = Lookup::new(®istry); + let ids = |name: &str, team: &str| -> Vec { + let routed = lookup.team_wildcard(name, team); + routed.iter().map(|d| d.id.clone()).collect() + }; + assert_eq!(ids("tw-gpt-4o", "T1"), ["tw1"]); + assert!(ids("gpt-4o", "T1").is_empty()); + assert!(ids("tw-gpt-4o", "T2").is_empty()); + } + + fn entry(name: &str, id: &str, info: Value) -> Value { + let mut info = info.as_object().cloned().unwrap_or_default(); + info.insert("id".to_owned(), json!(id)); + json!({"model_name": name, "litellm_params": {"model": format!("openai/{id}")}, "model_info": info}) + } + + fn at(position: usize) -> Row { + Row::At { + position, + name: None, + group: None, + } + } + + #[test] + fn all_prefers_the_teams_public_name_then_the_group() { + let registry = registry(json!([ + entry("gpt", "a", json!({})), + entry( + "team-gpt", + "t", + json!({"team_id": "T", "team_public_model_name": "gpt"}) + ), + entry("gpt", "b", json!({"team_id": "U"})), + ])); + let lookup = Lookup::new(®istry); + assert_eq!(lookup.all("gpt", Some("T"), None), vec![at(1)]); + assert_eq!(lookup.all("gpt", Some("V"), None), vec![at(0)]); + assert_eq!(lookup.all("gpt", None, None), vec![at(0), at(2)]); + assert_eq!(lookup.all("missing", None, None), vec![]); + } + + #[test] + fn a_junk_position_is_handed_back_to_raise() { + let mut registry = registry(json!([entry("gpt", "a", json!({}))])); + let mut model_name = registry.host_indexes().model_name.clone(); + model_name.insert("gpt".to_owned(), json!([0, "x"])); + let changes = crate::registry::ChangeSet { + host_index: Some(crate::host_index::IndexWrite { + model_name: Some(model_name), + ..Default::default() + }), + ..Default::default() + }; + registry = registry.apply(&changes, None).unwrap(); + let lookup = Lookup::new(®istry); + assert_eq!(lookup.all("gpt", None, None), vec![Row::Raw(json!("x"))]); + } + + #[test] + fn the_model_list_lists_aliases_and_groups_then_every_deployment() { + let registry = registry(json!([ + entry("gpt", "a", json!({"access_groups": ["g1"]})), + entry("claude", "b", json!({})), + ])); + let lookup = Lookup::new(®istry); + let scope = ListScope { + aliases: vec![AliasItem { + alias: "fast".to_owned(), + target: "gpt".to_owned(), + }], + groups: vec![CallableGroup { + name: "pool".to_owned(), + members: vec!["gpt".to_owned(), "claude".to_owned()], + }], + wildcard: Vec::new(), + only: None, + }; + let rows = lookup.model_list(None, None, &scope); + assert_eq!(rows.len(), 5); + assert_eq!( + rows[0], + Row::At { + position: 0, + name: Some("fast".to_owned()), + group: None + } + ); + assert_eq!( + lookup.model_names(None, &scope).unwrap(), + ["fast", "pool", "pool", "gpt", "claude"] + ); + assert_eq!( + lookup.access_groups(None, None, None, &scope).unwrap(), + vec![("g1".to_owned(), vec!["fast".to_owned(), "gpt".to_owned()])] + ); + } +} diff --git a/litellm-rust/crates/router-core/src/mock.rs b/litellm-rust/crates/router-core/src/mock.rs new file mode 100644 index 00000000000..898782e6203 --- /dev/null +++ b/litellm-rust/crates/router-core/src/mock.rs @@ -0,0 +1,44 @@ +//! The request's `mock_testing_*` flags: a failure the router raises itself, without calling a +//! deployment (`_handle_mock_testing_fallbacks` and `_handle_mock_testing_rate_limit_error`). + +use serde::Deserialize; + +use crate::error::{AttemptError, FailureKind}; + +/// Which mock failure a request asks for. +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::IntoStaticStr, strum::EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum MockFailure { + /// `mock_testing_fallbacks`: an `InternalServerError` before the first level's retries. + Fallbacks, + /// `mock_testing_context_fallbacks`: a `ContextWindowExceededError`, likewise. + ContextWindow, + /// `mock_testing_content_policy_fallbacks`: a `ContentPolicyViolationError`, likewise. + ContentPolicy, + /// `mock_testing_rate_limit_error`: a `RateLimitError` in place of every level's first + /// attempt, which the retry loop handles like a failed attempt. + RateLimit, +} + +impl MockFailure { + /// Whether the failure stands in for the first level's retry loop (raised before its retries). + pub fn before_retries(self) -> bool { + !matches!(self, Self::RateLimit) + } + + /// The failure as the rules see it. + pub fn failure(self, num_retries: Option) -> AttemptError { + let (kind, status) = match self { + Self::Fallbacks => (FailureKind::InternalServer, 500), + Self::ContextWindow => (FailureKind::ContextWindowExceeded, 400), + Self::ContentPolicy => (FailureKind::ContentPolicyViolation, 400), + Self::RateLimit => (FailureKind::RateLimit, 429), + }; + let mut failure = AttemptError::new(kind, Some(status)); + failure.num_retries = num_retries; + failure + } +} diff --git a/litellm-rust/crates/router-core/src/model_info.rs b/litellm-rust/crates/router-core/src/model_info.rs new file mode 100644 index 00000000000..861388f36ba --- /dev/null +++ b/litellm-rust/crates/router-core/src/model_info.rs @@ -0,0 +1,716 @@ +//! Model info: what the router reports a deployment can do and costs, merged from the process +//! cost map, what the background refresh discovered, and the deployment's own `model_info`. +//! +//! The merge rules are the Python router core's (`get_router_model_info`, +//! `get_deployment_model_info`); every cost-map read is a [`Host::model_info`] lookup asked while +//! the query runs, so a price reload or a registration the host made since the router was built is +//! what the next query reads. The host keeps choosing the keys a lookup tries (`get_model_info`); +//! the core chooses which lookups a query makes. +//! +//! Values the host cannot spell as JSON travel as opaque tokens ([`is_opaque`]): the merge moves +//! them between layers without reading them, and the host restores them. + +use serde_json::{Map, Number, Value}; + +use crate::{ + error::Error, + host::{Host, LogLevel, ModelInfoLookup}, +}; + +/// The key the host wraps a value it cannot spell as JSON in. +pub const OPAQUE_KEY: &str = "__litellm_python_value__"; + +/// A value the host passed as an opaque token: the merge moves it, never reads it. +pub fn is_opaque(value: &Value) -> bool { + matches!(value, Value::Object(map) if map.len() == 1 && map.contains_key(OPAQUE_KEY)) +} + +/// One `get_router_model_info` query, as the host reads it off the deployment. +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(default)] +pub struct RouterModelInfoQuery { + /// `model_info.id`. + pub deployment_id: Value, + /// `model_info.base_model`, then `litellm_params.base_model`: the first that is not `None` + /// names the model whose limits and costs the deployment reports. + pub model_info_base_model: Value, + pub params_base_model: Value, + /// The validated `litellm_params.model`. + pub params_model: String, + /// `base_model` among the validated params' extra fields. + pub extra_base_model: Value, + /// The name the caller asked for, which a wildcard deployment matches. + pub received_model_name: String, +} + +/// A merged model info, and the keys whose values are the deployment's own (the last layer). +#[derive(Clone, Debug, PartialEq)] +pub struct MergedModelInfo { + pub info: Map, + pub user_keys: Vec, +} + +pub(crate) fn raised(exception: &'static str, detail: impl Into) -> Error { + Error::ModelInfo { + exception, + detail: detail.into(), + } +} + +pub(crate) fn malformed(lookup: &str) -> Error { + raised( + "ValueError", + format!("the host answered a `{lookup}` lookup malformed"), + ) +} + +/// Python truthiness of a JSON value. +pub(crate) fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64().is_some_and(|n| n != 0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(map) => is_opaque(value) || !map.is_empty(), + } +} + +/// `(value or 0) > 0`, for a value that is a number (anything else reads as not positive). +fn positive_or_zero(value: Option<&Value>) -> bool { + value + .filter(|value| truthy(value)) + .and_then(Value::as_f64) + .is_some_and(|n| n > 0.0) +} + +pub(crate) async fn lookup(host: &dyn Host, lookup: ModelInfoLookup) -> Result { + host.model_info(&lookup).await +} + +/// `get_model_info(model=...)`: the row, or `None` when the lookup tolerates a failure and failed. +async fn catalog_row( + host: &dyn Host, + model: Value, + tolerate: bool, +) -> Result>, Error> { + match lookup( + host, + ModelInfoLookup::ModelInfo { + model, + tolerate, + exact: false, + }, + ) + .await? + { + Value::Null => Ok(None), + Value::Object(row) if tolerate && row.get("failed") == Some(&Value::Bool(true)) => Ok(None), + Value::Object(row) => Ok(Some(row)), + _ => Err(malformed("model_info")), + } +} + +/// `model_cost.get(key)`: `None` when the key is absent, else the entry as stored. +async fn cost_entry(host: &dyn Host, key: &str, process: bool) -> Result, Error> { + match lookup( + host, + ModelInfoLookup::CostEntry { + key: key.to_owned(), + process, + }, + ) + .await? + { + Value::Null => Ok(None), + Value::Object(mut answer) => Ok(Some(answer.remove("entry").unwrap_or(Value::Null))), + _ => Err(malformed("cost_entry")), + } +} + +pub(crate) async fn discovered(host: &dyn Host, id: Value) -> Result, Error> { + match lookup(host, ModelInfoLookup::Discovered { id }).await? { + Value::Null => Ok(Map::new()), + Value::Object(limits) => Ok(limits), + _ => Err(malformed("discovered")), + } +} + +async fn log(host: &dyn Host, level: LogLevel, message: String) -> Result<(), Error> { + lookup(host, ModelInfoLookup::Log { level, message }) + .await + .map(|_| ()) +} + +/// `_provider_model_id`: the model id a cost-map key or base model names, bedrock base model ids +/// normalized. +async fn provider_model_id(host: &dyn Host, name: &str, provider: &str) -> Result { + let model_id = name.rsplit('/').next().unwrap_or(name).to_owned(); + if !provider.starts_with("bedrock") { + return Ok(model_id); + } + match lookup(host, ModelInfoLookup::BedrockBase { model_id }).await? { + Value::String(base) => Ok(base), + _ => Err(malformed("bedrock_base")), + } +} + +/// `_exact_base_model_max_input_tokens`: the base model's own window, unless its row came from a +/// fallback-generalization rule or it is unmapped. +async fn exact_base_model_max_input_tokens( + host: &dyn Host, + base_model: &str, +) -> Result, Error> { + let answer = lookup( + host, + ModelInfoLookup::ModelInfo { + model: Value::String(base_model.to_owned()), + tolerate: true, + exact: true, + }, + ) + .await?; + let Value::Object(mut answer) = answer else { + return Ok(None); + }; + if answer.get("failed") == Some(&Value::Bool(true)) + || answer.get("generalized") == Some(&Value::Bool(true)) + { + return Ok(None); + } + let Some(Value::Object(row)) = answer.remove("row") else { + return Ok(None); + }; + Ok(row + .get("max_input_tokens") + .filter(|tokens| !tokens.is_null()) + .cloned()) +} + +/// `_base_model_widened_window`: a base model naming the same model as the route can only widen +/// its context window. +async fn base_model_widened_window( + host: &dyn Host, + route: &Map, + base_model: Option<&str>, + provider: &str, +) -> Result, Error> { + let Some(route_max) = route + .get("max_input_tokens") + .filter(|tokens| !tokens.is_null()) + else { + return Ok(None); + }; + let Some(base_model) = base_model else { + return Ok(None); + }; + let route_key = match route.get("key") { + Some(Value::String(key)) => key.clone(), + Some(_) => return Err(raised("AttributeError", "key")), + None => return Err(raised("KeyError", "key")), + }; + let route_id = provider_model_id(host, &route_key, provider).await?; + let base_id = provider_model_id(host, base_model, provider).await?; + let unversioned = base_id + .split_once('@') + .map_or(base_id.as_str(), |(id, _)| id); + if route_id != base_id && route_id != unversioned { + return Ok(None); + } + let Some(base_max) = exact_base_model_max_input_tokens(host, base_model).await? else { + return Ok(None); + }; + match (base_max.as_f64(), route_max.as_f64()) { + (Some(base), Some(route)) if base > route => Ok(Some(base_max)), + (Some(_), Some(_)) => Ok(None), + _ => Err(raised( + "TypeError", + "'<=' not supported between the base model's and the route's max_input_tokens", + )), + } +} + +/// `_declared_base_model`: a `base_model` that is a string. +fn declared_base_model(value: &Value) -> Option<&str> { + value.as_str() +} + +/// `get_router_model_info` after the host validated the deployment's params: the cost-map row of +/// the model the deployment serves, widened by its base model, overlaid with what the background +/// refresh discovered and then with every value the deployment sets itself. `None` when the +/// lookup answers `None`; an unmapped model raises as the lookup raises. +pub async fn router_model_info( + host: &dyn Host, + query: &RouterModelInfoQuery, +) -> Result, Error> { + let base_model = if query.model_info_base_model.is_null() { + query.params_base_model.clone() + } else { + query.model_info_base_model.clone() + }; + let mut model = base_model.clone(); + + let (resolved_model, provider) = match lookup( + host, + ModelInfoLookup::Provider { + model: query.params_model.clone(), + }, + ) + .await? + { + Value::Array(pair) => match pair.as_slice() { + [Value::String(model), Value::String(provider), ..] => { + (model.clone(), provider.clone()) + } + _ => return Err(malformed("provider")), + }, + _ => return Err(malformed("provider")), + }; + + if provider == "azure" && base_model.is_null() { + // Router init registers every deployment name into the cost map as a zeroed stub, so + // membership alone cannot tell a resolvable name apart; it needs usable limits or costs. + let fallback_key = if resolved_model.starts_with("azure/") { + resolved_model.clone() + } else { + format!("azure/{resolved_model}") + }; + let resolves = match cost_entry(host, &fallback_key, false).await? { + Some(Value::Object(entry)) if !is_opaque(&Value::Object(entry.clone())) => { + positive_or_zero(entry.get("max_input_tokens")) + || positive_or_zero(entry.get("input_cost_per_token")) + } + _ => false, + }; + if resolves { + log( + host, + LogLevel::Debug, + format!( + "Azure deployment '{resolved_model}' has no base_model set; using '{fallback_key}' from the model cost map for max tokens, cost tracking, etc." + ), + ) + .await?; + } else { + log( + host, + LogLevel::Error, + format!( + "Could not identify azure model '{resolved_model}'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models" + ), + ) + .await?; + } + } else if provider != "azure" { + model = Value::String(resolved_model.clone()); + if resolved_model.contains('*') { + // Only a wildcard deployment asks which model its pattern routed the name to. + if let Value::Array(routed) = lookup( + host, + ModelInfoLookup::PatternModels { + name: query.received_model_name.clone(), + }, + ) + .await? + && let Some(routed) = routed + .iter() + .find(|routed| routed.get("id").unwrap_or(&Value::Null) == &query.deployment_id) + { + model = routed.get("model").cloned().unwrap_or(Value::Null); + } + } + } + + if model.is_null() { + // An azure deployment without a base_model is looked up by its own model. + model = Value::String(resolved_model); + } + let Value::String(model) = model else { + return Err(raised( + "AttributeError", + format!( + "'{}' object has no attribute 'startswith'", + crate::group_info::type_name(&model) + ), + )); + }; + let model_info_name = if model.starts_with(&format!("{provider}/")) { + model + } else { + format!("{provider}/{model}") + }; + + let Some(row) = catalog_row(host, Value::String(model_info_name), false).await? else { + return Ok(None); + }; + + let user_model_info = match lookup(host, ModelInfoLookup::UserModelInfo).await? { + Value::Object(info) => info, + Value::Null => Map::new(), + _ => return Err(malformed("user_model_info")), + }; + let declared = user_model_info + .get("base_model") + .and_then(declared_base_model) + .or_else(|| declared_base_model(&query.extra_base_model)); + let widened = base_model_widened_window(host, &row, declared, &provider).await?; + let discovered = discovered(host, query.deployment_id.clone()).await?; + + let mut info = row; + if let Some(max_input_tokens) = widened { + info.insert("max_input_tokens".to_owned(), max_input_tokens); + } + info.extend(discovered); + let mut user_keys = Vec::new(); + for (key, value) in &user_model_info { + if !value.is_null() { + info.insert(key.clone(), value.clone()); + user_keys.push(key.clone()); + } + } + Ok(Some(MergedModelInfo { info, user_keys })) +} + +/// `_convert_stringified_numbers`: a string that reads as a number becomes one (a float when it +/// has an exponent or a point, else an int); any other string stays as written. +fn convert_stringified_number(text: &str) -> Value { + let trimmed = text.trim(); + let digits = without_digit_underscores(trimmed); + let lower = text.to_lowercase(); + if lower.contains('e') || text.contains('.') { + return digits + .and_then(|digits| python_float(&digits)) + .and_then(Number::from_f64) + .map_or_else(|| Value::String(text.to_owned()), Value::Number); + } + digits + .and_then(|digits| python_int(&digits)) + .map_or_else(|| Value::String(text.to_owned()), Value::Number) +} + +/// The text with the underscores Python's `int()`/`float()` accept between digits removed, or +/// `None` when an underscore sits anywhere else. +pub(crate) fn without_digit_underscores(text: &str) -> Option { + let chars: Vec = text.chars().collect(); + let mut out = String::with_capacity(text.len()); + for (index, ch) in chars.iter().enumerate() { + if *ch == '_' { + let before = index.checked_sub(1).and_then(|i| chars.get(i)); + let after = chars.get(index + 1); + if !(before.is_some_and(char::is_ascii_digit) + && after.is_some_and(char::is_ascii_digit)) + { + return None; + } + continue; + } + out.push(*ch); + } + Some(out) +} + +pub(crate) fn python_float(text: &str) -> Option { + let lower = text.to_ascii_lowercase(); + let unsigned = lower.trim_start_matches(['+', '-']); + if matches!(unsigned, "inf" | "infinity" | "nan") { + // Not finite: JSON cannot carry it, so the value stays as written. + return None; + } + let well_formed = text + .chars() + .all(|ch| ch.is_ascii_digit() || matches!(ch, '+' | '-' | '.' | 'e' | 'E')); + if !well_formed { + return None; + } + text.parse::().ok().filter(|n| n.is_finite()) +} + +pub(crate) fn python_int(text: &str) -> Option { + let unsigned = text.strip_prefix(['+', '-']).unwrap_or(text); + if unsigned.is_empty() || !unsigned.chars().all(|ch| ch.is_ascii_digit()) { + return None; + } + let text = text.strip_prefix('+').unwrap_or(text); + text.parse::() + .map(Number::from) + .ok() + .or_else(|| text.parse::().map(Number::from).ok()) +} + +/// `update_dictionary`: `new`'s values over `existing`, skipping `None`s; a string that reads as a +/// number becomes one, and a dict merges one level into the dict it replaces. +pub fn update_dictionary(existing: &mut Map, new: &Map) { + for (key, value) in new { + match value { + Value::Null => {} + Value::String(text) => { + existing.insert(key.clone(), convert_stringified_number(text)); + } + Value::Object(nested) if !is_opaque(value) => { + let merged = match existing.get(key) { + Some(current @ Value::Object(current_map)) if !is_opaque(current) => { + let mut merged = current_map.clone(); + merged.extend(nested.clone()); + merged + } + _ => nested.clone(), + }; + existing.insert(key.clone(), Value::Object(merged)); + } + _ => { + existing.insert(key.clone(), value.clone()); + } + } + } +} + +/// `cost_map_omits_token_price`: the cost-map entries under `keys` exist, and none carries a +/// per-token price. +async fn cost_map_omits_token_price(host: &dyn Host, keys: &[Option<&str>]) -> Result { + let mut entries = 0; + for key in keys.iter().flatten() { + if let Some(entry) = cost_entry(host, key, true).await? + && !is_opaque(&entry) + && let Value::Object(map) = entry + { + entries += 1; + if map.contains_key("input_cost_per_token") || map.contains_key("output_cost_per_token") + { + return Ok(false); + } + } + } + Ok(entries > 0) +} + +/// `get_deployment_model_info`: deployment `model_id`'s own cost-map entry (with what the refresh +/// discovered, over its base model's row), merged over the row of its litellm model `model_name`. +/// A lookup that fails leaves its layer out, as the Python router ignores it. +pub async fn deployment_model_info( + host: &dyn Host, + model_id: &str, + model_name: &Value, +) -> Result>, Error> { + let mut custom: Option> = None; + if let Some(entry) = cost_entry(host, model_id, false).await? { + let entry = if truthy(&entry) { + match entry { + Value::Object(map) if !is_opaque(&Value::Object(map.clone())) => Some(map), + // Not a mapping: the Python router's TypeError, which it ignores. + _ => None, + } + } else { + Some(Map::new()) + }; + if let Some(mut entry) = entry { + entry.extend(discovered(host, Value::String(model_id.to_owned())).await?); + custom = Some(entry); + } + } + + let builtin = catalog_row(host, model_name.clone(), true).await?; + + let mut base_model_key: Option = None; + if let Some(current) = &custom + && let Some(base_model) = current.get("base_model").filter(|base| !base.is_null()) + && let Some(mut base_row) = catalog_row(host, base_model.clone(), true).await? + { + base_model_key = base_row + .get("key") + .and_then(Value::as_str) + .map(str::to_owned); + // The base model provides defaults; the deployment's own entry overrides them. + update_dictionary(&mut base_row, current); + custom = Some(base_row); + } + + let builtin_key = builtin + .as_ref() + .and_then(|row| row.get("key")) + .and_then(Value::as_str) + .map(str::to_owned); + let info = match (custom, builtin) { + (Some(custom), Some(mut builtin)) => { + update_dictionary(&mut builtin, &custom); + builtin + } + (None, Some(builtin)) => builtin, + (Some(custom), None) => custom, + (None, None) => return Ok(None), + }; + if cost_map_omits_token_price( + host, + &[ + Some(model_id), + builtin_key.as_deref(), + base_model_key.as_deref(), + ], + ) + .await? + { + let mut info = info; + info.insert("input_cost_per_token".to_owned(), Value::Null); + info.insert("output_cost_per_token".to_owned(), Value::Null); + return Ok(Some(info)); + } + Ok(Some(info)) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + }; + + use async_trait::async_trait; + use serde_json::json; + + use super::*; + use crate::{ + cache::{InMemoryRouterCache, RouterCache}, + error::AttemptError, + host::{Attempt, Output, RouterEvent, RouterRequest, RoutingContext, SelectStage}, + registry::Deployment, + }; + + /// A cost map and a provider, answering lookups the way `PythonHost` does. + #[derive(Default)] + struct Catalog { + rows: BTreeMap, + provider: (String, String), + user_model_info: Value, + lookups: Mutex>, + } + + #[async_trait] + impl Host for Catalog { + async fn invoke(&self, _attempt: &Attempt) -> Result { + unreachable!("a model-info query runs no attempt") + } + async fn select( + &self, + _stage: &SelectStage, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + Ok(candidates) + } + async fn allow_fallback( + &self, + _request: &RouterRequest, + _target: &str, + ) -> Result { + Ok(true) + } + async fn sleep(&self, _duration: std::time::Duration) {} + fn observe(&self, _event: &RouterEvent) {} + fn cache(&self) -> Arc { + Arc::new(InMemoryRouterCache::default()) + } + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + self.lookups + .lock() + .unwrap() + .push(serde_json::to_value(lookup).unwrap()); + Ok(match lookup { + ModelInfoLookup::ModelInfo { + model, tolerate, .. + } => match model.as_str().and_then(|model| self.rows.get(model)) { + Some(row) => row.clone(), + None if *tolerate => json!({"failed": true}), + None => return Err(raised("ValueError", "unmapped")), + }, + ModelInfoLookup::CostEntry { key, .. } => self + .rows + .get(key) + .map_or(Value::Null, |entry| json!({"entry": entry})), + ModelInfoLookup::Provider { .. } => { + json!([self.provider.0, self.provider.1]) + } + ModelInfoLookup::UserModelInfo => self.user_model_info.clone(), + _ => Value::Null, + }) + } + } + + #[test] + fn update_dictionary_converts_numbers_and_merges_nested_dicts_one_level() { + let mut existing = json!({"a": 1, "n": {"x": 1, "y": 2}, "k": "keep"}) + .as_object() + .unwrap() + .clone(); + let new = json!({"a": None::, "n": {"y": 3}, "i": "8_192", "f": "3e-07", "s": "abc"}) + .as_object() + .unwrap() + .clone(); + update_dictionary(&mut existing, &new); + assert_eq!( + Value::Object(existing), + json!({"a": 1, "n": {"x": 1, "y": 3}, "k": "keep", "i": 8192, "f": 3e-07, "s": "abc"}) + ); + } + + #[tokio::test] + async fn router_model_info_overlays_the_deployments_own_values_on_the_row() { + let host = Catalog { + rows: BTreeMap::from([( + "openai/gpt-4o".to_owned(), + json!({"key": "gpt-4o", "max_input_tokens": 100, "input_cost_per_token": 1.0}), + )]), + provider: ("gpt-4o".to_owned(), "openai".to_owned()), + user_model_info: json!({"id": "d", "input_cost_per_token": 2.0, "mode": None::}), + ..Catalog::default() + }; + let query = RouterModelInfoQuery { + deployment_id: json!("d"), + params_model: "openai/gpt-4o".to_owned(), + received_model_name: "g".to_owned(), + ..RouterModelInfoQuery::default() + }; + let merged = router_model_info(&host, &query).await.unwrap().unwrap(); + assert_eq!( + Value::Object(merged.info), + json!({"key": "gpt-4o", "max_input_tokens": 100, "input_cost_per_token": 2.0, "id": "d"}) + ); + assert_eq!(merged.user_keys, vec!["id", "input_cost_per_token"]); + } + + #[tokio::test] + async fn deployment_model_info_merges_the_entry_over_the_builtin_row() { + let host = Catalog { + rows: BTreeMap::from([ + ( + "gpt-4o".to_owned(), + json!({"key": "gpt-4o", "max_tokens": 10, "input_cost_per_token": 1.0}), + ), + ("d".to_owned(), json!({"max_tokens": "20"})), + ]), + ..Catalog::default() + }; + let info = deployment_model_info(&host, "d", &json!("gpt-4o")) + .await + .unwrap() + .unwrap(); + assert_eq!( + Value::Object(info), + json!({"key": "gpt-4o", "max_tokens": 20, "input_cost_per_token": 1.0}) + ); + } + + #[tokio::test] + async fn an_unpriced_entry_reports_no_token_prices() { + let host = Catalog { + rows: BTreeMap::from([("d".to_owned(), json!({"max_tokens": 5}))]), + ..Catalog::default() + }; + let info = deployment_model_info(&host, "d", &json!("unmapped")) + .await + .unwrap() + .unwrap(); + assert_eq!( + Value::Object(info), + json!({"max_tokens": 5, "input_cost_per_token": null, "output_cost_per_token": null}) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/outcome.rs b/litellm-rust/crates/router-core/src/outcome.rs new file mode 100644 index 00000000000..59e0d14b2c2 --- /dev/null +++ b/litellm-rust/crates/router-core/src/outcome.rs @@ -0,0 +1,125 @@ +//! Which of a request's outcomes reach the deployment's health. The Python router learns them from +//! logging callbacks it registers on the process lists; `_core_serve_through_request_logging` and +//! `_core_discard` take those off, and `_core_bind_to_request_logging` puts them back on one +//! request's logging object. This core registers none, so the host answers, for each request, what +//! would have run ([`OutcomeCallbacks`]), and every write a callback owns passes through +//! [`OutcomeCallbacks::record`]. +//! +//! What the Python router writes outside its callbacks is not gated, and reaches [`crate::Health`] +//! with the default value: `make_call`'s usage, `async_routing_strategy_pre_call_checks`' +//! cooldown, `_trigger_cooldown_for_failed_deployment`, `RouterHealth.record_failure` and a +//! realtime session's success. + +use std::future::Future; + +use serde::Deserialize; + +use crate::error::Error; + +/// Which of the router's outcome callbacks litellm's logging runs for a request. The default is a +/// router whose callbacks are on the process lists. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct OutcomeCallbacks { + /// `deployment_callback_on_success`, `sync_deployment_callback_on_success` for a sync call. + /// `_core_bind_to_request_logging` binds only the former. + pub success: bool, + /// `deployment_callback_on_failure` and `async_deployment_callback_on_failure`. + pub failure: bool, + /// The attempt runs through `make_call`, which counts its usage whatever the callbacks do. + pub make_call: bool, +} + +impl Default for OutcomeCallbacks { + fn default() -> Self { + Self { + success: true, + failure: true, + make_call: true, + } + } +} + +/// What an outcome writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutcomeWrite { + /// The success counter, or the usage the success callback adds at the end of a stream. + Success, + /// The tokens and request of an attempt that returned: `make_call`'s write, or the success + /// callback's for an endpoint that skips `make_call`. + AttemptUsage, + /// The failure counter, the allowed-fails counter, the cooldown and the failed request. + Failure, +} + +impl OutcomeCallbacks { + /// Run `record` when `write` happens for this request, else record nothing. + pub(crate) async fn record( + self, + write: OutcomeWrite, + record: impl Future>, + ) -> Result { + let runs = match write { + OutcomeWrite::Success => self.success, + OutcomeWrite::AttemptUsage => self.success || self.make_call, + OutcomeWrite::Failure => self.failure, + }; + if runs { record.await } else { Ok(T::default()) } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const REGISTERED: OutcomeCallbacks = OutcomeCallbacks { + success: true, + failure: true, + make_call: true, + }; + const UNBOUND: OutcomeCallbacks = OutcomeCallbacks { + success: false, + failure: false, + make_call: true, + }; + const UNBOUND_WITHOUT_MAKE_CALL: OutcomeCallbacks = OutcomeCallbacks { + success: false, + failure: false, + make_call: false, + }; + const BOUND_SYNC: OutcomeCallbacks = OutcomeCallbacks { + success: false, + failure: true, + make_call: true, + }; + + #[rstest] + #[case::registered_success(REGISTERED, OutcomeWrite::Success, true)] + #[case::registered_usage(REGISTERED, OutcomeWrite::AttemptUsage, true)] + #[case::registered_failure(REGISTERED, OutcomeWrite::Failure, true)] + #[case::unbound_success(UNBOUND, OutcomeWrite::Success, false)] + #[case::unbound_usage(UNBOUND, OutcomeWrite::AttemptUsage, true)] + #[case::unbound_failure(UNBOUND, OutcomeWrite::Failure, false)] + #[case::no_make_call_usage(UNBOUND_WITHOUT_MAKE_CALL, OutcomeWrite::AttemptUsage, false)] + #[case::bound_sync_success(BOUND_SYNC, OutcomeWrite::Success, false)] + #[case::bound_sync_failure(BOUND_SYNC, OutcomeWrite::Failure, true)] + #[tokio::test] + async fn a_write_runs_when_its_callback_does( + #[case] callbacks: OutcomeCallbacks, + #[case] write: OutcomeWrite, + #[case] runs: bool, + ) { + let recorded = callbacks.record(write, async { Ok(true) }).await; + + assert_eq!(recorded.ok(), Some(runs)); + } + + #[test] + fn a_router_on_the_process_lists_is_the_default() { + let parsed: OutcomeCallbacks = serde_json::from_str("{}").expect("an empty document"); + + assert_eq!(parsed, REGISTERED); + } +} diff --git a/litellm-rust/crates/router-core/src/precall.rs b/litellm-rust/crates/router-core/src/precall.rs new file mode 100644 index 00000000000..04dd22f1c90 --- /dev/null +++ b/litellm-rust/crates/router-core/src/precall.rs @@ -0,0 +1,300 @@ +//! `enable_pre_call_checks` (`_pre_call_checks`): before a pick, drop the deployments whose +//! context window the prompt exceeds, those outside the request's `allowed_model_region`, and +//! those that cannot take a requested param, then decide which error a group with nothing left +//! raises. The facts that need live model data (the prompt's token count, each deployment's +//! context window, which params it takes) come from [`Host::pre_call_facts`]; every decision is +//! made here. +//! +//! [`Host::pre_call_facts`]: crate::host::Host::pre_call_facts + +use std::{collections::BTreeMap, sync::Arc}; + +use serde::Deserialize; +use serde_json::Value; + +use crate::{error::Error, registry::Deployment}; + +/// What the host knows about one deployment for the pre-call checks. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +#[serde(default)] +pub struct DeploymentFacts { + /// The deployment's context window (`get_router_model_info`'s `max_input_tokens`), when it + /// declares one and resolves. + pub max_input_tokens: Option, + /// `_deployment_lacks_params`: it does not support one of the requested params. + pub lacks_params: bool, + /// `_deployment_rejects_params`: it lacks one the request and deployment neither drop nor allow. + pub rejects_params: bool, +} + +/// What the host knows about a request for the pre-call checks. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +#[serde(default)] +pub struct PreCallFacts { + /// The request carries `messages` or `input`: without either no check runs. + pub has_input: bool, + /// The prompt may be counted against a context window (no compaction is pending). + pub countable: bool, + /// The prompt's token count, when it was counted. + pub input_tokens: Option, + /// The request's `allowed_model_region`. + pub allowed_model_region: Option, + /// The requested params the check filters on, sorted (`_pre_call_check_requested_chat_params`). + pub requested_params: Vec, + /// Facts by deployment id, for the candidates and the group's deployments. + pub deployments: BTreeMap, +} + +impl PreCallFacts { + fn of(&self, deployment: &Deployment) -> DeploymentFacts { + self.deployments + .get(&deployment.id) + .cloned() + .unwrap_or_default() + } +} + +/// The name a context-window error gives a deployment: its base model, else its model. +fn deployment_model(deployment: &Deployment) -> String { + let truthy = |value: Option<&Value>| match value { + Some(Value::String(text)) if !text.is_empty() => Some(text.clone()), + Some(Value::Null | Value::Bool(false)) | None => None, + Some(Value::String(_)) => None, + Some(other) => Some(python_str(other)), + }; + truthy(deployment.model_info.raw.get("base_model")) + .or_else(|| truthy(deployment.raw_params.get("base_model"))) + .unwrap_or_else(|| deployment.params.model.clone()) +} + +/// `str(value)` for the scalar values a base model may hold. +fn python_str(value: &Value) -> String { + match value { + Value::Bool(true) => "True".to_owned(), + Value::Number(number) => number.to_string(), + other => other.to_string(), + } +} + +/// `is_region_allowed`: the deployment's `region_name` equals the allowed region. +fn in_region(deployment: &Deployment, allowed: &Value) -> bool { + deployment.raw_params.get("region_name") == Some(allowed) +} + +/// `_pre_call_checks` over `candidates`; `pool` is the group's deployments before health filters +/// (`request_deployments`), which decide whether nothing left is an unsupported-params error. +pub fn filter( + model: &str, + candidates: Vec>, + pool: &[Arc], + facts: &PreCallFacts, +) -> Result>, Error> { + if !facts.has_input { + return Ok(candidates); + } + let mut invalid = vec![false; candidates.len()]; + let mut context_window_error = false; + let mut detail = String::new(); + for (index, deployment) in candidates.iter().enumerate() { + let found = facts.of(deployment); + if let (Some(limit), true, Some(tokens)) = + (found.max_input_tokens, facts.countable, facts.input_tokens) + && tokens > limit + { + invalid[index] = true; + context_window_error = true; + detail.push_str(&format!( + "Model={}, Max Input Tokens={limit}, Got={tokens}", + deployment_model(deployment) + )); + continue; + } + if let Some(allowed) = &facts.allowed_model_region + && !allowed.is_null() + && !in_region(deployment, allowed) + { + invalid[index] = true; + } + } + let params_checked = !facts.requested_params.is_empty(); + let param_candidates: Vec = (0..candidates.len()).filter(|i| !invalid[*i]).collect(); + let lacking: Vec = param_candidates + .iter() + .map(|i| params_checked && facts.of(&candidates[*i]).lacks_params) + .collect(); + let none_takes = !lacking.is_empty() && lacking.iter().all(|lacks| *lacks); + let failing: Vec = if none_takes { + param_candidates + .iter() + .map(|i| params_checked && facts.of(&candidates[*i]).rejects_params) + .collect() + } else { + lacking + }; + for (index, fails) in param_candidates.iter().zip(failing) { + if fails { + invalid[*index] = true; + } + } + let excluded = invalid.iter().filter(|x| **x).count(); + if excluded == candidates.len() { + if !context_window_error + && !pool.is_empty() + && pool + .iter() + .all(|d| params_checked && facts.of(d).rejects_params) + { + return Err(Error::UnsupportedParams { + model: model.to_owned(), + params: facts.requested_params.clone(), + }); + } + if context_window_error { + return Err(Error::ContextWindowExceeded { + model: model.to_owned(), + detail, + }); + } + } + Ok(candidates + .into_iter() + .zip(invalid) + .filter_map(|(deployment, invalid)| (!invalid).then_some(deployment)) + .collect()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn deployment(id: &str, params: Value, model_info: Value) -> Arc { + let mut info = model_info; + info["id"] = json!(id); + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": params, "model_info": info, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + fn facts(entries: &[(&str, DeploymentFacts)]) -> PreCallFacts { + PreCallFacts { + has_input: true, + countable: true, + deployments: entries + .iter() + .map(|(id, f)| ((*id).to_owned(), f.clone())) + .collect(), + ..PreCallFacts::default() + } + } + + fn window(limit: i64) -> DeploymentFacts { + DeploymentFacts { + max_input_tokens: Some(limit), + ..DeploymentFacts::default() + } + } + + #[rstest] + fn a_prompt_over_a_context_window_drops_that_deployment() { + let small = deployment("small", json!({"model": "openai/a"}), json!({})); + let big = deployment("big", json!({"model": "openai/b"}), json!({})); + let mut found = facts(&[("small", window(5)), ("big", window(10_000))]); + found.input_tokens = Some(100); + let kept = filter("g", vec![small, big], &[], &found).unwrap(); + assert_eq!( + kept.iter().map(|d| d.id.as_str()).collect::>(), + ["big"] + ); + } + + #[rstest] + fn no_window_left_names_each_deployment_back_to_back() { + let one = deployment("one", json!({"model": "openai/one"}), json!({})); + let two = deployment( + "two", + json!({"model": "openai/two"}), + json!({"base_model": "azure/base"}), + ); + let mut found = facts(&[("one", window(5)), ("two", window(6))]); + found.input_tokens = Some(9); + let error = filter("g", vec![one, two], &[], &found).unwrap_err(); + let Error::ContextWindowExceeded { model, detail } = error else { + panic!("expected a context window error, got {error:?}"); + }; + assert_eq!(model, "g"); + assert_eq!( + detail, + "Model=openai/one, Max Input Tokens=5, Got=9Model=azure/base, Max Input Tokens=6, Got=9" + ); + } + + #[rstest] + fn without_input_nothing_is_checked() { + let small = deployment("small", json!({"model": "openai/a"}), json!({})); + let mut found = facts(&[("small", window(5))]); + found.input_tokens = Some(100); + found.has_input = false; + assert_eq!(filter("g", vec![small], &[], &found).unwrap().len(), 1); + } + + #[rstest] + #[case::in_region("eu", 1)] + #[case::out_of_region("ap", 0)] + fn the_allowed_region_keeps_only_its_deployments(#[case] region: &str, #[case] kept: usize) { + let eu = deployment( + "eu", + json!({"model": "openai/a", "region_name": "eu"}), + json!({}), + ); + let none = deployment("none", json!({"model": "openai/a"}), json!({})); + let mut found = facts(&[]); + found.allowed_model_region = Some(json!(region)); + assert_eq!( + filter("g", vec![eu, none], &[], &found).unwrap().len(), + kept + ); + } + + #[rstest] + fn a_group_where_every_deployment_rejects_the_param_is_unsupported() { + let old = deployment("old", json!({"model": "openai/old"}), json!({})); + let mut found = facts(&[( + "old", + DeploymentFacts { + lacks_params: true, + rejects_params: true, + ..DeploymentFacts::default() + }, + )]); + found.requested_params = vec!["response_format".to_owned()]; + let error = filter("g", vec![old.clone()], &[old], &found).unwrap_err(); + assert!( + matches!(error, Error::UnsupportedParams { params, .. } if params == ["response_format"]) + ); + } + + #[rstest] + fn a_deployment_that_takes_the_param_wins_over_one_that_would_drop_it() { + let old = deployment("old", json!({"model": "openai/old"}), json!({})); + let new = deployment("new", json!({"model": "openai/new"}), json!({})); + let mut found = facts(&[( + "old", + DeploymentFacts { + lacks_params: true, + ..DeploymentFacts::default() + }, + )]); + found.requested_params = vec!["response_format".to_owned()]; + let kept = filter("g", vec![old, new], &[], &found).unwrap(); + assert_eq!( + kept.iter().map(|d| d.id.as_str()).collect::>(), + ["new"] + ); + } +} diff --git a/litellm-rust/crates/router-core/src/prerouting.rs b/litellm-rust/crates/router-core/src/prerouting.rs new file mode 100644 index 00000000000..d6e3df05e7a --- /dev/null +++ b/litellm-rust/crates/router-core/src/prerouting.rs @@ -0,0 +1,53 @@ +//! Pre-routing: before an async pick resolves its name, the host may route the request to another +//! model group (Python's `_core_async_pre_routing_hook`). The complexity, auto, adaptive and +//! quality routers registered from `auto_router/...` marker deployments, and the Claude Code +//! session binding, live host-side; the core asks once per pick, before resolution, and resolves +//! the name the host answers. The attempt still serves the requested name (its `model_group`): +//! only the deployments come from the routed group. +//! +//! A custom routing strategy (`register_plugin`, `set_custom_routing_strategy`, lar1) replaces the +//! whole pick (`_core_async_get_available_deployment` and `get_available_deployment`): the host +//! answers the same step with the deployment document the strategy returned, which may name a +//! deployment the registry does not hold, and the core serves that deployment as is. + +use std::sync::Arc; + +use crate::{Core, config::Op, error::Error, host::RouterRequest, registry::Deployment}; + +/// What the host's pre-routing step sees. +#[derive(Clone, Copy, Debug)] +pub struct PreRoute<'a> { + pub op: &'a Op, + pub request: &'a RouterRequest, + /// The name this pick resolves: the requested name or the fallback target. + pub name: &'a str, + /// How many fallbacks deep the chain is (Python's `fallback_depth`): only a first attempt + /// binds a Claude Code session. + pub fallback_depth: u32, + /// 0 for the first attempt in this group, then the retry number. + pub retry: u32, +} + +/// The host's answer to a pre-routing step. +#[derive(Clone, Debug)] +pub enum PreRouted { + /// Resolve this model group instead of the requested name. + Model(String), + /// A custom strategy's pick: serve this deployment, without resolving or filtering. + Deployment(Arc), +} + +impl Core { + /// What a pick of `ask.name` serves: the host's pre-routed group or custom pick, else the + /// name itself. Only picks that ask for it pre-route (`request.pre_routes`). + pub(crate) async fn pre_routed(&self, ask: PreRoute<'_>) -> Result { + if !ask.request.pre_routes { + return Ok(PreRouted::Model(ask.name.to_owned())); + } + Ok(self + .host + .pre_route(&ask) + .await? + .unwrap_or_else(|| PreRouted::Model(ask.name.to_owned()))) + } +} diff --git a/litellm-rust/crates/router-core/src/ptu.rs b/litellm-rust/crates/router-core/src/ptu.rs new file mode 100644 index 00000000000..9110ac6a67f --- /dev/null +++ b/litellm-rust/crates/router-core/src/ptu.rs @@ -0,0 +1,96 @@ +//! PTU shares (`litellm_core_utils/ptu_shares.py`): a deployment whose `model_info.ptu_shares` +//! splits it across teams serves only those teams, and the teams holding a share are kept to +//! their share. + +use std::{collections::BTreeSet, sync::Arc}; + +use serde_json::{Map, Value}; + +use crate::registry::Deployment; + +/// `parsed_ptu_shares`: the teams of a non-empty `ptu_shares` map whose every entry is a team id +/// and a positive whole number of PTUs (`2.0` reads as 2, a bool is no number), else None. +pub fn share_teams(model_info: &Map) -> Option> { + let shares = model_info.get("ptu_shares")?.as_object()?; + if shares.is_empty() { + return None; + } + shares + .iter() + .map(|(team, share)| (!team.is_empty() && whole_ptus(share)).then(|| team.clone())) + .collect() +} + +/// `_whole_ptus`: a positive whole number. +fn whole_ptus(share: &Value) -> bool { + match share { + Value::Number(number) => number.as_i64().map_or_else( + || { + number + .as_u64() + .map(|whole| whole > 0) + .or_else(|| number.as_f64().map(|f| f.fract() == 0.0 && f > 0.0)) + .unwrap_or(false) + }, + |whole| whole > 0, + ), + _ => false, + } +} + +/// `filter_ptu_shared_deployments`: the deployments `team_id` holds a share on when it holds one, +/// else the ones split into no shares; a caller without a team holds none. Also whether any +/// deployment was withheld. +pub fn filter_shared( + deployments: Vec>, + team_id: Option<&str>, +) -> (Vec>, bool) { + let team = team_id.unwrap_or(""); + let checks: Vec<(Arc, Option>)> = deployments + .into_iter() + .map(|deployment| { + let shares = share_teams(&deployment.model_info.raw); + (deployment, shares) + }) + .collect(); + let holds_share = checks + .iter() + .any(|(_, shares)| shares.as_ref().is_some_and(|teams| teams.contains(team))); + let total = checks.len(); + let kept: Vec> = checks + .into_iter() + .filter(|(_, shares)| match shares { + Some(teams) => teams.contains(team), + None => !holds_share, + }) + .map(|(deployment, _)| deployment) + .collect(); + let withheld = kept.len() < total; + (kept, withheld) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case(json!({"TA": 50}), Some(vec!["TA"]))] + #[case(json!({"TA": 2.0, "TB": 1}), Some(vec!["TA", "TB"]))] + #[case(json!({"TA": 2.5}), None)] + #[case(json!({"TA": true}), None)] + #[case(json!({"TA": 0}), None)] + #[case(json!({}), None)] + #[case(json!({"": 1}), None)] + #[case(json!("TA"), None)] + fn reads_the_share_teams(#[case] shares: Value, #[case] expected: Option>) { + let mut model_info = Map::new(); + model_info.insert("ptu_shares".to_owned(), shares); + assert_eq!( + share_teams(&model_info), + expected.map(|teams| teams.into_iter().map(str::to_owned).collect()) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/registry.rs b/litellm-rust/crates/router-core/src/registry.rs new file mode 100644 index 00000000000..78c7d3b67e3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/registry.rs @@ -0,0 +1,1548 @@ +//! The configured deployments, indexed by id, model group, alias, wildcard pattern, routing +//! group and team. A [`Registry`] is immutable; [`Registry::apply`] builds the next version. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + fmt::Write as _, + sync::Arc, +}; + +use litellm_router_types::{LitellmParams, Spelled}; +use serde::Deserialize; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use crate::{ + error::ConfigError, + host_index::{HostIndexes, IndexWrite, cache_flags_left_to_the_catalog, deployment_model_name}, +}; + +/// A registry's version; every change produces a higher one. +pub type RegistryVersion = u64; + +/// A deployment as a config or an `apply` spells it. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +pub struct DeploymentSpec { + pub model_name: String, + pub litellm_params: Map, + #[serde(default)] + pub model_info: Map, + /// The entry's other top-level keys (a config's `tpm`, ...), kept so the host's view of the + /// deployment list reads back the entries it stored. + #[serde(flatten)] + pub extra: Map, + /// The provider the host resolved when it built the deployment ([`crate::build`]); never + /// read from a config or an `apply`. + #[serde(skip)] + pub provider: Option, +} + +/// `routing_groups`: a named subset of model groups sharing a routing strategy. +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct RoutingGroupSpec { + pub group_name: String, + pub models: Vec, + pub routing_strategy: crate::config::RoutingStrategy, + /// A priority group's member priorities, served as each member's `order`. + #[serde(default)] + pub model_priorities: Option>, +} + +/// The `model_info` fields the router reads; every other key stays in [`Self::raw`]. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ModelInfo { + pub team_id: Option, + pub team_public_model_name: Option, + pub access_groups: Vec, + pub blocked: bool, + pub cooldown_time: Option, + pub allowed_fails: Option, + /// `model_info.allowed_fails_policy` when it is a dict, read as Python reads it: field name + /// to allowed fails, values as given. + pub allowed_fails_policy: Option>, + pub raw: Map, +} + +impl ModelInfo { + fn from_raw(raw: Map) -> Self { + let text = |key: &str| raw.get(key).and_then(Value::as_str).map(str::to_owned); + Self { + team_id: text("team_id"), + team_public_model_name: text("team_public_model_name"), + access_groups: raw + .get("access_groups") + .and_then(Value::as_array) + .map(|groups| { + groups + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default(), + blocked: raw.get("blocked").and_then(Value::as_bool) == Some(true), + cooldown_time: raw.get("cooldown_time").and_then(Value::as_f64), + allowed_fails: raw + .get("allowed_fails") + .and_then(Value::as_u64) + .and_then(|fails| u32::try_from(fails).ok()), + allowed_fails_policy: raw + .get("allowed_fails_policy") + .and_then(Value::as_object) + .cloned(), + raw, + } + } +} + +/// One configured deployment. +#[derive(Clone, Debug, PartialEq)] +pub struct Deployment { + pub id: String, + pub model_name: String, + pub params: LitellmParams, + /// `litellm_params` as written, in order: what the id hashes and the host receives. + pub raw_params: Map, + pub model_info: ModelInfo, + /// The entry's other top-level keys, as written. + pub extra: Map, + /// The provider the host resolved for it, when it was built through the host. + pub resolved_provider: Option, +} + +impl Deployment { + pub fn from_spec(spec: DeploymentSpec) -> Result { + let params: LitellmParams = serde_json::from_value(Value::Object( + spec.litellm_params.clone(), + )) + .map_err(|source| ConfigError::InvalidDeployment { + model_name: spec.model_name.clone(), + source, + })?; + let id = match spec.model_info.get("id") { + Some(Value::String(id)) => id.clone(), + Some(Value::Number(id)) => id.to_string(), + _ => deployment_id(&spec.model_name, &spec.litellm_params), + }; + Ok(Self { + id, + model_name: spec.model_name, + params, + raw_params: spec.litellm_params, + model_info: ModelInfo::from_raw(spec.model_info), + extra: spec.extra, + resolved_provider: spec.provider, + }) + } + + /// The deployment as a `model_list` entry: its other top-level keys, then `model_name`, + /// `litellm_params` and `model_info` (with its id). + pub fn record(&self) -> Value { + let mut model_info = self.model_info.raw.clone(); + model_info.insert("id".to_owned(), Value::String(self.id.clone())); + let mut record = self.extra.clone(); + record.insert( + "model_name".to_owned(), + Value::String(self.model_name.clone()), + ); + record.insert( + "litellm_params".to_owned(), + Value::Object(self.raw_params.clone()), + ); + record.insert("model_info".to_owned(), Value::Object(model_info)); + Value::Object(record) + } + + /// The stand-in an unrouted attempt carries: the requested name as its group and model, no id. + pub fn unrouted(name: &str) -> Result { + let mut params = Map::new(); + params.insert("model".to_owned(), Value::String(name.to_owned())); + let mut info = Map::new(); + info.insert("id".to_owned(), Value::String(String::new())); + Self::from_spec(DeploymentSpec { + model_name: name.to_owned(), + litellm_params: params, + model_info: info, + extra: Map::new(), + provider: None, + }) + } + + /// The provider: the one the host resolved when it built the deployment, else + /// `custom_llm_provider`, else the `model` prefix before `/`. + pub fn provider(&self) -> &str { + self.resolved_provider + .as_deref() + .or(self.params.custom_llm_provider.as_deref()) + .or_else(|| { + self.params + .model + .split_once('/') + .map(|(provider, _)| provider) + }) + .unwrap_or("openai") + } + + /// A numeric `litellm_params` value, as `simple_shuffle._metric_weight` reads it. + pub fn metric(&self, key: &str) -> Option { + let spelled = |value: &Option>| match value { + Some(Spelled::Value(value)) => Some(*value), + Some(Spelled::Text(_)) | None => None, + }; + match key { + "rpm" => spelled(&self.params.rpm), + "tpm" => spelled(&self.params.tpm), + _ => self.raw_params.get(key).and_then(Value::as_f64), + } + } + + /// `simple_shuffle._metric_weight`: 0 when unset, a number (a bool counts as 0 or 1), or + /// `None` when the value is not numeric. A `rpm`/`tpm` spelled as text reads as unset. + pub fn metric_weight(&self, key: &str) -> Option { + if key != "weight" { + return Some(self.metric(key).unwrap_or(0.0)); + } + match self.raw_params.get(key) { + None | Some(Value::Null) => Some(0.0), + Some(Value::Bool(flag)) => Some(f64::from(u8::from(*flag))), + Some(Value::Number(number)) => number.as_f64(), + Some(_) => None, + } + } + + /// `get_deployment_order`: `litellm_params.order`, else `model_info.order`, which is where a + /// team model added through the API keeps it. Neither is validated as an int, so a bool, a + /// whole float and a numeric string are read as the level they compare as. + pub fn order(&self) -> Option { + let order = self + .raw_params + .get("order") + .filter(|order| !order.is_null()) + .or_else(|| self.model_info.raw.get("order"))?; + match order { + Value::Bool(flag) => Some(i64::from(*flag)), + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .filter(|float| float.fract() == 0.0) + .map(|float| float as i64) + }), + Value::String(text) => text.parse().ok(), + Value::Null | Value::Array(_) | Value::Object(_) => None, + } + } + + /// `_deployment_supports_web_search`: `model_info.supports_web_search` when it is set, read + /// for truthiness, else true. + pub(crate) fn supports_web_search(&self) -> bool { + self.model_info + .raw + .get("supports_web_search") + .is_none_or(|flag| flag.is_null() || crate::host_index::truthy(flag)) + } + + /// Whether `team_id` may use this deployment (`deployment_usable_by_team`). + pub fn usable_by_team(&self, team_id: Option<&str>) -> bool { + self.model_info + .team_id + .as_deref() + .is_none_or(|owner| Some(owner) == team_id) + } + + /// Whether `_core_get_model_list(team_id=team_id)` lists this deployment: no team lists + /// every team's (`_core_should_include_deployment`). + pub fn listed_for_team(&self, team_id: Option<&str>) -> bool { + team_id.is_none() || self.usable_by_team(team_id) + } + + /// Whether the deployment declares a limit the router counts per-minute usage for + /// (`_deployment_tracks_usage` and `deployment_has_io_token_limits`, without the limits only + /// the model catalog knows, which the host decides by): `tpm` or `rpm`, `itpm` or `otpm`, on + /// the entry itself, in `litellm_params` or in `model_info`. + pub fn tracks_usage(&self) -> bool { + let declared = |section: &Map| { + ["tpm", "rpm", "itpm", "otpm"] + .iter() + .any(|key| section.get(*key).is_some_and(|limit| !limit.is_null())) + }; + self.params.tpm.is_some() + || self.params.rpm.is_some() + || self.params.itpm.is_some() + || self.params.otpm.is_some() + || declared(&self.extra) + || declared(&self.model_info.raw) + } +} + +/// `_return_pattern_matched_deployments` for one deployment: a copy serving `request`, its +/// `litellm_params.model` filled with the `groups` the pattern matched and the prompt-cache flags +/// left to the catalog. A deployment whose model holds no wildcard is served as it is. +pub(crate) fn routed_deployment( + deployment: &Arc, + groups: &[String], + request: &str, +) -> Arc { + let model = &deployment.params.model; + if !model.contains('*') { + return deployment.clone(); + } + let named = deployment_model_name(groups, request, model); + let mut routed = Deployment::clone(deployment); + for flag in cache_flags_left_to_the_catalog(&deployment.raw_params) { + routed.model_info.raw.remove(*flag); + } + routed + .raw_params + .insert("model".to_owned(), Value::String(named.clone())); + routed.params.model = named; + Arc::new(routed) +} + +/// The connection params of one deployment (`credentials(id)`). +#[derive(Clone, Debug, PartialEq)] +pub struct Credentials { + pub provider: String, + pub api_base: Option, + pub api_version: Option, + pub credential_name: Option, + pub params: LitellmParams, +} + +/// What a name resolved through. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResolveKind { + SpecificDeployment, + Id, + /// A name nothing else serves, routed to the router's `default_deployment` (a copy whose + /// `litellm_params.model` is the name). Served like a deployment named by its id. + Default, + Alias, + RoutingGroup, + Group, + Wildcard, +} + +impl ResolveKind { + /// The name resolved to one deployment that is served as is: a deployment id, or the + /// default deployment (`_common_checks_available_deployment` returns a dict for both). + pub fn is_single(&self) -> bool { + matches!(self, Self::Id | Self::Default) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Resolution { + pub kind: ResolveKind, + /// The model group the deployments are routed as. + pub group: String, + pub deployments: Vec>, +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct ResolveScope { + /// The caller named a deployment's `litellm_params.model` (`specific_deployment=True`). + pub specific_deployment: bool, +} + +#[derive(Clone, Debug, Default)] +pub struct DeploymentFilter<'a> { + pub model_name: Option<&'a str>, + pub team_id: Option<&'a str>, + pub provider: Option<&'a str>, +} + +#[derive(Clone, Debug, Default)] +pub struct NameFilter { + pub include_aliases: bool, +} + +/// The registry indexes as the host wrote them over the ones the deployments build (the +/// deprecated `model_names`, `model_name_to_deployment_indices` and +/// `team_model_to_deployment_indices` writes), which resolution reads instead from then on. +/// Indices point into the deployment list; one past its end is skipped. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct IndexOverrides { + /// `_core_model_names`: the names `name in model_names` finds. + pub model_names: Option>, + /// `_core_model_name_to_deployment_indices`: a model name's deployments. + pub model_name: Option>>, + /// `_core_team_model_to_deployment_indices`, as `(team_id, public name, indices)` entries. + pub team_model: Option)>>, +} + +impl IndexOverrides { + fn is_empty(&self) -> bool { + self.model_names.is_none() && self.model_name.is_none() && self.team_model.is_none() + } +} + +/// One atomic change: optionally a whole new deployment list, then upserts, deletes and, +/// optionally, a new dynamic config. +#[derive(Clone, Debug, Default)] +pub struct ChangeSet { + /// Every deployment, in order, replacing the current list (the host registry's `model_list` + /// after a write; it may hold one id more than once, as the Python router's list can). + pub replace: Option>, + /// Deployments added at the end, beside any that share their id (a deployment the host + /// retries after its provider resolved). + pub appends: Vec, + pub upserts: Vec, + pub deletes: Vec, + pub settings: Option, + /// The indexes the host wrote ([`IndexOverrides`]), replacing the ones held; an empty one + /// goes back to the indexes the deployments build. + pub index: Option, + /// The deployment at the position the id map holds for this id, removed with every index + /// shifted (`_delete_deployment`, and an upsert's removal of the deployment it edits). + pub remove: Option, + /// A host write to the indexes it hands out ([`HostIndexes`]). + pub host_index: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct Registry { + version: RegistryVersion, + /// The version the deployments before the newest appends last changed at: a host that read + /// the list at or after it reads only what was appended since ([`Self::deployments_from`]). + rewritten: RegistryVersion, + deployments: Vec>, + by_id: HashMap, + by_group: BTreeMap>, + by_team: HashMap>, + wildcards: Vec<(String, Vec)>, + aliases: BTreeMap, + routing_groups: Vec, + overrides: Option>, + /// The indexes the host hands out, kept by the Python router's history rules. + host: Arc, + /// `_model_to_group` as Python last built it, when a raw `routing_groups` write replaced the + /// groups since: member calls keep routing by it (see [`Self::member_routing_group`]). + member_groups: Option>>, +} + +impl Registry { + pub fn build( + specs: Vec, + aliases: BTreeMap, + routing_groups: Vec, + ) -> Result { + // One id may name several deployments (a per-organization fan-out, or two entries + // declaring one id): each is kept, and the id finds the last, as the Python router keeps them. + let deployments = specs + .into_iter() + .map(|spec| Deployment::from_spec(spec).map(Arc::new)) + .collect::, _>>()?; + // The indexes `set_model_list` leaves, as a build installing each deployment in turn does. + let mut host = HostIndexes::default(); + for (position, deployment) in deployments.iter().enumerate() { + let record = deployment.record(); + host.added( + record.as_object().expect("a record is an object"), + position, + false, + ); + } + host.set_model_names(deployments.iter().map(|d| d.model_name.clone()).collect()); + Ok(Self { + host: Arc::new(host), + ..Self::index(0, deployments, aliases, routing_groups) + }) + } + + fn index( + version: RegistryVersion, + deployments: Vec>, + aliases: BTreeMap, + routing_groups: Vec, + ) -> Self { + let mut by_id = HashMap::new(); + let mut by_group: BTreeMap> = BTreeMap::new(); + let mut by_team: HashMap> = HashMap::new(); + for (index, deployment) in deployments.iter().enumerate() { + by_id.insert(deployment.id.clone(), index); + by_group + .entry(deployment.model_name.clone()) + .or_default() + .push(index); + if let Some(team) = &deployment.model_info.team_id { + by_team.entry(team.clone()).or_default().push(index); + } + } + // In the order each pattern first appears, then most specific first, as + // `PatternUtils.sorted_patterns` keeps them: a stable sort, longest regex first, + // then the most regex syntax. + let mut wildcards: Vec<(String, Vec)> = Vec::new(); + for deployment in &deployments { + let name = &deployment.model_name; + if name.contains('*') && !wildcards.iter().any(|(pattern, _)| pattern == name) { + wildcards.push(( + name.clone(), + by_group.get(name).cloned().unwrap_or_default(), + )); + } + } + wildcards.sort_by_key(|(pattern, _)| std::cmp::Reverse(pattern_specificity(pattern))); + Self { + version, + rewritten: version, + deployments, + by_id, + by_group, + by_team, + wildcards, + aliases, + routing_groups, + overrides: None, + host: Arc::default(), + member_groups: None, + } + } + + /// This registry with the stale routing-group member map (`None` derives it from the groups). + #[must_use] + pub fn with_member_groups(self, member_groups: Option>) -> Self { + Self { + member_groups: member_groups.map(Arc::new), + ..self + } + } + + pub fn version(&self) -> RegistryVersion { + self.version + } + + /// The version the deployments before the newest appends last changed at (see `rewritten`). + pub fn rewritten(&self) -> RegistryVersion { + self.rewritten + } + + /// Every deployment id, in list order (`names(kind="ids")`). + pub fn ids(&self) -> Vec { + self.deployments.iter().map(|d| d.id.clone()).collect() + } + + /// The deployments from position `start` on, in list order. + pub fn deployments_from(&self, start: usize) -> &[Arc] { + self.deployments.get(start..).unwrap_or_default() + } + + /// The next registry: upserts, then deletes (`apply_changes`). An upserted deployment whose + /// id exists is removed and appended at the end, as the Python router does on an edit. + pub fn apply( + &self, + changes: &ChangeSet, + aliases: Option>, + ) -> Result { + let mut deployments = match &changes.replace { + Some(specs) => specs + .iter() + .map(|spec| self.carried(spec.clone()).map(Arc::new)) + .collect::, _>>()?, + None => self.deployments.clone(), + }; + for spec in &changes.appends { + deployments.push(Arc::new(self.carried(spec.clone())?)); + } + for spec in &changes.upserts { + let deployment = self.carried(spec.clone())?; + deployments.retain(|existing| existing.id != deployment.id); + deployments.push(Arc::new(deployment)); + } + deployments.retain(|deployment| !changes.deletes.contains(&deployment.id)); + let mut host = HostIndexes::clone(&self.host); + if let Some(id) = &changes.remove + && let Some(position) = host.position_of(id) + && position < deployments.len() + { + deployments.remove(position); + host.removed(id, position) + .map_err(|error| ConfigError::Removal(error.to_string()))?; + } + if let Some(write) = &changes.host_index { + host.write(write.clone()); + } + let overrides = match &changes.index { + Some(index) => (!index.is_empty()).then(|| Arc::new(index.clone())), + None => self.overrides.clone(), + }; + // Appends alone leave every deployment before them as it was. + let appended_only = changes.replace.is_none() + && changes.upserts.is_empty() + && changes.deletes.is_empty() + && changes.remove.is_none(); + let next = Self::index( + self.version + 1, + deployments, + aliases.unwrap_or_else(|| self.aliases.clone()), + self.routing_groups.clone(), + ); + Ok(Self { + overrides, + host: Arc::new(host), + member_groups: self.member_groups.clone(), + rewritten: if appended_only { + self.rewritten + } else { + next.rewritten + }, + ..next + }) + } + + /// The next registry for a build starting over (`set_model_list`): no deployments, and the + /// indexes [`HostIndexes::reset`] clears cleared. + #[must_use] + pub fn emptied(&self) -> Self { + let mut host = HostIndexes::clone(&self.host); + host.reset(); + Self { + overrides: self.overrides.clone(), + host: Arc::new(host), + member_groups: self.member_groups.clone(), + ..Self::index( + self.version + 1, + Vec::new(), + self.aliases.clone(), + self.routing_groups.clone(), + ) + } + } + + /// Add one deployment a build accepted at the end and index it in place (`add` is + /// `add_deployment`, which also names its group). A build installs into a registry of its + /// own this way, so a list of n deployments builds in O(n) rather than reindexing per + /// deployment. + pub fn push_built(&mut self, spec: DeploymentSpec, add: bool) -> Result<(), ConfigError> { + let deployment = Arc::new(self.carried(spec)?); + let index = self.deployments.len(); + let record = deployment.record(); + Arc::make_mut(&mut self.host).added( + record.as_object().expect("a record is an object"), + index, + add, + ); + self.by_id.insert(deployment.id.clone(), index); + let name = &deployment.model_name; + self.by_group.entry(name.clone()).or_default().push(index); + if let Some(team) = &deployment.model_info.team_id { + self.by_team.entry(team.clone()).or_default().push(index); + } + if name.contains('*') { + match self + .wildcards + .iter_mut() + .find(|(pattern, _)| pattern == name) + { + Some((_, indices)) => indices.push(index), + None => { + // A new pattern joins after every one it ties with, as `index` orders them: + // first appearance, then a stable sort by specificity. + self.wildcards.push((name.clone(), vec![index])); + self.wildcards.sort_by_key(|(pattern, _)| { + std::cmp::Reverse(pattern_specificity(pattern)) + }); + } + } + } + self.deployments.push(deployment); + self.version += 1; + Ok(()) + } + + /// The last deployment added. + pub fn last(&self) -> Option<&Arc> { + self.deployments.last() + } + + /// Take `set_model_list`'s `model_names`: the names it was given. + pub fn set_model_names(&mut self, names: Vec) { + Arc::make_mut(&mut self.host).set_model_names(names); + self.version += 1; + } + + /// The indexes the host hands out. + pub fn host_indexes(&self) -> &HostIndexes { + &self.host + } + + /// The deployment `spec` describes, keeping the provider the host resolved for the + /// deployment it replaces (same id, same params) when the spec carries none: a host that + /// hands back its whole list after a write does not resolve every provider again. + fn carried(&self, mut spec: DeploymentSpec) -> Result { + let deployment = Deployment::from_spec(spec.clone())?; + if spec.provider.is_none() + && let Some(previous) = self + .by_id + .get(&deployment.id) + .and_then(|index| self.deployments.get(*index)) + && previous.raw_params == deployment.raw_params + && previous.model_name == deployment.model_name + { + spec.provider.clone_from(&previous.resolved_provider); + return Deployment::from_spec(spec); + } + Ok(deployment) + } + + /// This registry with the providers `from` resolved for the same deployments (same id, + /// same params) and `from`'s host indexes: a router rebuilt for new static settings over the + /// same list keeps what its host resolved and the indexes their history built. + #[must_use] + pub fn with_providers_of(&self, from: &Self) -> Self { + let deployments = self + .deployments + .iter() + .map(|deployment| { + let previous = from + .by_id + .get(&deployment.id) + .and_then(|index| from.deployments.get(*index)) + .filter(|previous| { + previous.raw_params == deployment.raw_params + && previous.model_name == deployment.model_name + }); + match previous { + Some(previous) if deployment.resolved_provider.is_none() => { + Arc::new(Deployment { + resolved_provider: previous.resolved_provider.clone(), + ..Deployment::clone(deployment) + }) + } + _ => deployment.clone(), + } + }) + .collect(); + Self { + deployments, + host: from.host.clone(), + ..self.clone() + } + } + + fn at(&self, indices: &[usize]) -> Vec> { + indices + .iter() + .filter_map(|index| self.deployments.get(*index).cloned()) + .collect() + } + + /// `_core_deployments_named(name)`: the deployments the model-name index holds for `name` + /// (whatever their own name), or `None` when it does not index the name. + pub(crate) fn named(&self, name: &str) -> Option>> { + match self.overrides.as_ref().and_then(|o| o.model_name.as_ref()) { + Some(index) => index.get(name).map(|indices| self.at(indices)), + None => self.by_group.get(name).map(|indices| self.at(indices)), + } + } + + /// Whether the model-name index holds `name` (`name in model_name_to_deployment_indices`). + fn indexes_name(&self, name: &str) -> bool { + match self.overrides.as_ref().and_then(|o| o.model_name.as_ref()) { + Some(index) => index.contains_key(name), + None => self.by_group.contains_key(name), + } + } + + /// `_team_deployments_named(team_id, name)`: the deployments the team index holds for the + /// pair, or `None` when it does not index it. + pub(crate) fn team_named(&self, team_id: &str, name: &str) -> Option>> { + match self.overrides.as_ref().and_then(|o| o.team_model.as_ref()) { + Some(entries) => entries + .iter() + .find(|(team, public, _)| team == team_id && public == name) + .map(|(_, _, indices)| self.at(indices)), + None => { + let own: Vec> = self + .deployments + .iter() + .filter(|d| { + d.model_info.team_id.as_deref() == Some(team_id) + && d.model_info.team_public_model_name.as_deref() == Some(name) + }) + .cloned() + .collect(); + (!own.is_empty()).then_some(own) + } + } + } + + /// Every deployment the team index holds under public name `name`, any team's + /// (`_team_deployments_across_teams`). + pub(crate) fn team_named_across_teams(&self, name: &str) -> Vec> { + match self.overrides.as_ref().and_then(|o| o.team_model.as_ref()) { + Some(entries) => entries + .iter() + .filter(|(_, public, _)| public == name) + .flat_map(|(_, _, indices)| self.at(indices)) + .collect(), + None => self + .deployments + .iter() + .filter(|d| { + d.model_info.team_id.is_some() + && d.model_info.team_public_model_name.as_deref() == Some(name) + }) + .cloned() + .collect(), + } + } + + /// Whether a written index entry names a position past the model list. + fn overflows(&self, indices: &[usize]) -> bool { + indices.iter().any(|index| *index >= self.deployments.len()) + } + + /// The written model-name index entry for `name` names a position past the model list. + pub(crate) fn name_index_overflows(&self, name: &str) -> bool { + self.overrides + .as_ref() + .and_then(|o| o.model_name.as_ref()) + .and_then(|index| index.get(name)) + .is_some_and(|indices| self.overflows(indices)) + } + + /// The written team index entry for `(team_id, name)` names a position past the model list. + pub(crate) fn team_index_overflows(&self, team_id: &str, name: &str) -> bool { + self.overrides + .as_ref() + .and_then(|o| o.team_model.as_ref()) + .and_then(|entries| { + entries + .iter() + .find(|(team, public, _)| team == team_id && public == name) + }) + .is_some_and(|(_, _, indices)| self.overflows(indices)) + } + + /// A written team index entry under public name `name` names a position past the list. + pub(crate) fn across_teams_index_overflows(&self, name: &str) -> bool { + self.overrides + .as_ref() + .and_then(|o| o.team_model.as_ref()) + .is_some_and(|entries| { + entries + .iter() + .any(|(_, public, indices)| public == name && self.overflows(indices)) + }) + } + + /// The group a pick of `name` reads its strategy from, when the stale member map names a + /// group that is no longer configured (`_get_routing_context` raises `KeyError`): `name` + /// after its alias, unless it is a callable routing group. + pub fn missing_member_group(&self, name: &str) -> Option<&str> { + let members = self.member_groups.as_ref()?; + let resolved = self.alias_of(name).unwrap_or(name); + if self.routing_group_named(resolved).is_some() { + return None; + } + let group = members.get(resolved)?; + (!self.routing_groups.iter().any(|g| g.group_name == *group)).then_some(group.as_str()) + } + + /// The routing group a member call of `model_group` routes by: the stale member map's + /// group when there is one (`_model_to_group`), else the non-priority group listing it. + pub(crate) fn member_routing_group(&self, model_group: &str) -> Option<&RoutingGroupSpec> { + match &self.member_groups { + Some(members) => members + .get(model_group) + .and_then(|name| self.routing_groups.iter().find(|g| g.group_name == *name)), + None => self + .routing_groups + .iter() + .filter(|group| !crate::routing_groups::is_priority(group)) + .find(|group| group.models.iter().any(|model| model == model_group)), + } + } + + pub fn deployment(&self, id: &str) -> Option> { + self.by_id + .get(id) + .map(|index| self.deployments[*index].clone()) + } + + pub fn deployments(&self, filter: &DeploymentFilter<'_>) -> Vec> { + let candidates = match (filter.model_name, filter.team_id) { + (Some(name), _) => self + .by_group + .get(name) + .map(|i| self.at(i)) + .unwrap_or_default(), + (None, Some(team)) => self + .by_team + .get(team) + .map(|i| self.at(i)) + .unwrap_or_default(), + (None, None) => self.deployments.clone(), + }; + candidates + .into_iter() + .filter(|d| { + filter + .team_id + .is_none_or(|team| d.model_info.team_id.as_deref() == Some(team)) + }) + .filter(|d| { + filter + .provider + .is_none_or(|provider| d.provider() == provider) + }) + .collect() + } + + /// The model group names, in name order, with aliases when asked. + pub fn names(&self, filter: &NameFilter) -> Vec { + let mut names: Vec = self.by_group.keys().cloned().collect(); + if filter.include_aliases { + names.extend(self.aliases.keys().cloned()); + } + names + } + + pub fn group(&self, name: &str) -> Option>> { + if self + .overrides + .as_ref() + .is_some_and(|o| o.model_name.is_some()) + { + return self.named(name).map(|deployments| { + deployments + .into_iter() + .filter(|d| d.model_name == name) + .collect() + }); + } + self.by_group.get(name).map(|indices| self.at(indices)) + } + + /// How many rows `_core_get_model_list(model_name=name)` lists: the group's deployments, + /// an alias's target group, a routing group's members, and only when none of those, the + /// deployments of the wildcard the name matches. + pub fn model_list_len(&self, name: &str) -> usize { + let count = |group: &str| self.group(group).map_or(0, |deployments| deployments.len()); + let listed = count(name) + + self.aliases.get(name).map_or(0, |target| count(target)) + + self + .routing_groups + .iter() + .find(|group| group.group_name == name) + .map_or(0, |group| { + group.models.iter().map(|model| count(model)).sum() + }); + if listed > 0 { + return listed; + } + self.wildcards + .iter() + .find(|(pattern, _)| wildcard_matches(pattern, name)) + .map_or(0, |(_, indices)| indices.len()) + } + + /// The routing group a model group belongs to, if any. + pub fn routing_group_of(&self, model_name: &str) -> Option<&RoutingGroupSpec> { + self.routing_groups + .iter() + .find(|group| group.models.iter().any(|model| model == model_name)) + } + + /// The strategy a routing group named `group_name` declares. + pub fn resolve_routing_group_strategy( + &self, + group_name: &str, + ) -> Option { + self.routing_groups + .iter() + .find(|group| group.group_name == group_name) + .map(|group| group.routing_strategy.clone()) + } + + pub(crate) fn routing_groups(&self) -> &[RoutingGroupSpec] { + &self.routing_groups + } + + /// Every deployment, in configured order. + pub(crate) fn all(&self) -> &[Arc] { + &self.deployments + } + + /// Whether `name` is a deployment `model_name` (`name in _core_model_names`). + pub(crate) fn has_name(&self, name: &str) -> bool { + match self.overrides.as_ref().and_then(|o| o.model_names.as_ref()) { + Some(names) => names.contains(name), + None => self.by_group.contains_key(name), + } + } + + /// The group `model_group_alias` rewrites `name` to. + pub(crate) fn alias_of(&self, name: &str) -> Option<&str> { + self.aliases.get(name).map(String::as_str) + } + + /// The callable routing group named `name`: a model group of that name shadows it. + pub(crate) fn routing_group_named(&self, name: &str) -> Option<&RoutingGroupSpec> { + (!self.indexes_name(name)) + .then(|| self.routing_groups.iter().find(|g| g.group_name == name)) + .flatten() + } + + /// Whether any deployment is listed under a wildcard pattern. + pub(crate) fn has_wildcards(&self) -> bool { + !self.wildcards.is_empty() + } + + /// The most specific wildcard pattern matching `name`, with its deployments. + pub(crate) fn wildcard_for(&self, name: &str) -> Option<(String, Vec>)> { + self.wildcards + .iter() + .find(|(pattern, _)| wildcard_matches(pattern, name)) + .map(|(pattern, indices)| (pattern.clone(), self.at(indices))) + } + + /// Resolve a requested name, in the order of `_common_checks_available_deployment`: + /// a specific deployment, an id, an alias, a routing group, a model group, a wildcard. + pub fn resolve(&self, name: &str, scope: ResolveScope) -> Option { + let resolution = |kind, group: &str, deployments: Vec>| { + (!deployments.is_empty()).then(|| Resolution { + kind, + group: group.to_owned(), + deployments, + }) + }; + if scope.specific_deployment { + let matching = self + .deployments + .iter() + .filter(|d| d.params.model == name) + .cloned() + .collect(); + // An empty match still resolves: the pick then finds no deployment rather than + // falling back to a default (`_common_checks_available_deployment` returns `[]`). + return Some(Resolution { + kind: ResolveKind::SpecificDeployment, + group: name.to_owned(), + deployments: matching, + }); + } + if !self.by_group.contains_key(name) + && let Some(deployment) = self.deployment(name) + { + let group = deployment.model_name.clone(); + return resolution(ResolveKind::Id, &group, vec![deployment]); + } + let routing_group_named = |name: &str| { + // A model group of the same name shadows a routing group (`get_routing_group`). + (!self.by_group.contains_key(name)) + .then(|| self.routing_groups.iter().find(|g| g.group_name == name)) + .flatten() + }; + if let Some(target) = self.aliases.get(name) { + if let Some(routing_group) = routing_group_named(target) { + let deployments = crate::routing_groups::members(routing_group, |model| { + self.by_group + .get(model) + .map(|indices| self.at(indices)) + .unwrap_or_default() + }); + return resolution(ResolveKind::RoutingGroup, target, deployments); + } + return self + .group(target) + .and_then(|deployments| resolution(ResolveKind::Alias, target, deployments)); + } + if let Some(routing_group) = routing_group_named(name) { + let deployments = crate::routing_groups::members(routing_group, |model| { + self.by_group + .get(model) + .map(|indices| self.at(indices)) + .unwrap_or_default() + }); + return resolution(ResolveKind::RoutingGroup, name, deployments); + } + if let Some(indices) = self.by_group.get(name) { + return resolution(ResolveKind::Group, name, self.at(indices)); + } + self.wildcards + .iter() + .find(|(pattern, _)| wildcard_matches(pattern, name)) + .and_then(|(pattern, indices)| { + resolution(ResolveKind::Wildcard, pattern, self.at(indices)) + }) + } + + /// Access group name to the model groups in it. + pub fn access_groups(&self) -> BTreeMap> { + let mut groups: BTreeMap> = BTreeMap::new(); + for deployment in &self.deployments { + for group in &deployment.model_info.access_groups { + groups + .entry(group.clone()) + .or_default() + .insert(deployment.model_name.clone()); + } + } + groups + } + + pub fn credentials(&self, id: &str) -> Option { + self.deployment(id).map(|deployment| Credentials { + provider: deployment.provider().to_owned(), + api_base: deployment.params.api_base.clone(), + api_version: deployment.params.api_version.clone(), + credential_name: deployment.params.litellm_credential_name.clone(), + params: deployment.params.clone(), + }) + } +} + +/// `PatternUtils.calculate_pattern_specificity` over the pattern's regex +/// (`re.escape(pattern).replace(r"\*", "(.*)")`): its length, then how much regex syntax it has. +fn pattern_specificity(pattern: &str) -> (usize, usize) { + let mut regex = String::new(); + for c in pattern.chars() { + match c { + '*' => regex.push_str("(.*)"), + '(' | ')' | '[' | ']' | '{' | '}' | '?' | '+' | '-' | '|' | '^' | '$' | '\\' | '.' + | '&' | '~' | '#' | ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c' => { + regex.push('\\'); + regex.push(c); + } + _ => regex.push(c), + } + } + let complexity = regex + .chars() + .filter(|c| matches!(c, '*' | '+' | '?' | '\\' | '^' | '$' | '|' | '(' | ')')) + .count(); + (regex.chars().count(), complexity) +} + +/// `*` matches any run of characters, as the pattern router's `(.*)` does under `re.match`: +/// anchored at the start only, so the name may run on past the pattern's last literal. +fn wildcard_matches(pattern: &str, name: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or_default(); + let Some(mut rest) = name.strip_prefix(first) else { + return false; + }; + let parts: Vec<&str> = parts.collect(); + for part in parts { + match rest.find(part) { + Some(at) => rest = &rest[at + part.len()..], + None => return false, + } + } + true +} + +/// `deployment_id` in `litellm/router_api.py`: sha256 over the model group and every +/// `litellm_params` key and value in order. A string is taken as is, a dict as `json.dumps`, and +/// anything else as Python's `str()`. +pub fn deployment_id(model_group: &str, litellm_params: &Map) -> String { + let mut text = String::from(model_group); + for (key, value) in litellm_params { + text.push_str(key); + match value { + Value::String(value) => text.push_str(value), + Value::Object(_) => json_dumps(value, &mut text), + Value::Null | Value::Bool(_) | Value::Number(_) | Value::Array(_) => { + python_str(value, &mut text); + } + } + } + let digest = Sha256::digest(text.as_bytes()); + let mut hex = String::with_capacity(64); + for byte in digest { + let _ = write!(hex, "{byte:02x}"); + } + hex +} + +/// Python's `repr(float)`: the shortest round-trip digits, positional for exponents in +/// `-4..16`, scientific otherwise. +fn python_float(value: f64, out: &mut String) { + if value.is_nan() { + out.push_str("nan"); + return; + } + if value.is_infinite() { + out.push_str(if value > 0.0 { "inf" } else { "-inf" }); + return; + } + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific.split_once('e').unwrap_or((&scientific, "0")); + let exponent: i32 = exponent.parse().unwrap_or(0); + let (sign, mantissa) = mantissa + .strip_prefix('-') + .map_or(("", mantissa), |m| ("-", m)); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + out.push_str(sign); + if (-4..16).contains(&exponent) { + if exponent < 0 { + out.push_str("0."); + out.extend(std::iter::repeat_n('0', (-exponent - 1) as usize)); + out.push_str(&digits); + } else { + let point = exponent as usize + 1; + if digits.len() <= point { + out.push_str(&digits); + out.extend(std::iter::repeat_n('0', point - digits.len())); + out.push_str(".0"); + } else { + out.push_str(&digits[..point]); + out.push('.'); + out.push_str(&digits[point..]); + } + } + } else { + out.push_str(&digits[..1]); + if digits.len() > 1 { + out.push('.'); + out.push_str(&digits[1..]); + } + let _ = write!( + out, + "e{}{:02}", + if exponent < 0 { '-' } else { '+' }, + exponent.abs() + ); + } +} + +fn number(value: &serde_json::Number, out: &mut String) { + match value.as_f64() { + Some(float) if !value.is_i64() && !value.is_u64() => python_float(float, out), + Some(_) | None => out.push_str(&value.to_string()), + } +} + +/// `json.dumps` with its defaults: `", "` and `": "` separators and `ensure_ascii`. +fn json_dumps(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(flag) => out.push_str(if *flag { "true" } else { "false" }), + Value::Number(value) => number(value, out), + Value::String(text) => json_string(text, out), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + json_dumps(item, out); + } + out.push(']'); + } + Value::Object(entries) => { + out.push('{'); + for (index, (key, item)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + json_string(key, out); + out.push_str(": "); + json_dumps(item, out); + } + out.push('}'); + } + } +} + +fn json_string(text: &str, out: &mut String) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{08}' => out.push_str("\\b"), + '\u{0c}' => out.push_str("\\f"), + ch if ch.is_ascii() && !ch.is_ascii_control() => out.push(ch), + ch => { + let mut units = [0u16; 2]; + for unit in ch.encode_utf16(&mut units) { + let _ = write!(out, "\\u{unit:04x}"); + } + } + } + } + out.push('"'); +} + +/// Python's `str()` of a JSON value: `repr` for the items of a list or dict. +fn python_str(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("None"), + Value::Bool(flag) => out.push_str(if *flag { "True" } else { "False" }), + Value::Number(value) => number(value, out), + Value::String(text) => python_repr_str(text, out), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + python_str(item, out); + } + out.push(']'); + } + Value::Object(entries) => { + out.push('{'); + for (index, (key, item)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + python_repr_str(key, out); + out.push_str(": "); + python_str(item, out); + } + out.push('}'); + } + } +} + +fn python_repr_str(text: &str, out: &mut String) { + let quote = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + out.push(quote); + for ch in text.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + ch if ch == quote => { + out.push('\\'); + out.push(ch); + } + ch if (ch as u32) < 0x20 || ch as u32 == 0x7f => { + let _ = write!(out, "\\x{:02x}", ch as u32); + } + ch => out.push(ch), + } + } + out.push(quote); +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn spec(model_name: &str, params: Value, info: Value) -> DeploymentSpec { + serde_json::from_value(json!({ + "model_name": model_name, "litellm_params": params, "model_info": info, + })) + .unwrap() + } + + #[rstest] + #[case::none(json!({}), json!({}), json!({}), false)] + #[case::params_rpm(json!({"rpm": 1}), json!({}), json!({}), true)] + #[case::params_itpm(json!({"itpm": 1}), json!({}), json!({}), true)] + #[case::model_info_rpm(json!({}), json!({"rpm": 1}), json!({}), true)] + #[case::model_info_tpm(json!({}), json!({"tpm": 1}), json!({}), true)] + #[case::model_info_otpm(json!({}), json!({"otpm": 1}), json!({}), true)] + #[case::model_info_null(json!({}), json!({"rpm": null}), json!({}), false)] + #[case::top_level_tpm(json!({}), json!({}), json!({"tpm": 1}), true)] + fn a_limit_anywhere_on_the_deployment_tracks_usage( + #[case] params: Value, + #[case] info: Value, + #[case] entry: Value, + #[case] tracks: bool, + ) { + let mut spec = json!({ + "model_name": "g", + "litellm_params": {"model": "openai/a"}, + "model_info": {"id": "a"}, + }); + let extend = |section: &mut Value, more: Value| { + section + .as_object_mut() + .unwrap() + .extend(more.as_object().unwrap().clone()); + }; + extend(&mut spec["litellm_params"], params); + extend(&mut spec["model_info"], info); + extend(&mut spec, entry); + let spec: DeploymentSpec = serde_json::from_value(spec).unwrap(); + assert_eq!(Deployment::from_spec(spec).unwrap().tracks_usage(), tracks); + } + + #[rstest] + #[case::params_int(json!({"model": "m", "order": 2}), json!({}), Some(2))] + #[case::params_win(json!({"model": "m", "order": 2}), json!({"order": 1}), Some(2))] + #[case::model_info(json!({"model": "m"}), json!({"order": 3}), Some(3))] + #[case::params_null(json!({"model": "m", "order": null}), json!({"order": 3}), Some(3))] + #[case::numeric_string(json!({"model": "m", "order": "4"}), json!({}), Some(4))] + #[case::model_info_string(json!({"model": "m"}), json!({"order": "5"}), Some(5))] + #[case::whole_float(json!({"model": "m", "order": 6.0}), json!({}), Some(6))] + #[case::none(json!({"model": "m"}), json!({}), None)] + fn order_reads_as_get_deployment_order( + #[case] params: Value, + #[case] info: Value, + #[case] expected: Option, + ) { + let deployment = Deployment::from_spec(spec("g", params, info)).unwrap(); + assert_eq!(deployment.order(), expected); + } + + #[rstest] + #[case::unset(json!({}), true)] + #[case::declared(json!({"supports_web_search": true}), true)] + #[case::refused(json!({"supports_web_search": false}), false)] + #[case::null_is_unset(json!({"supports_web_search": null}), true)] + fn web_search_support_is_assumed_unless_declared(#[case] info: Value, #[case] expected: bool) { + let deployment = Deployment::from_spec(spec("g", json!({"model": "m"}), info)).unwrap(); + assert_eq!(deployment.supports_web_search(), expected); + } + + #[test] + fn a_routed_wildcard_fills_its_model_and_drops_the_cache_flags() { + let deployment = Arc::new( + Deployment::from_spec(spec( + "anthropic/*", + json!({"model": "anthropic/*"}), + json!({"id": "w", "supports_prompt_caching": true, "keep": 1}), + )) + .unwrap(), + ); + let routed = routed_deployment( + &deployment, + &["claude-3-haiku".to_owned()], + "anthropic/claude-3-haiku", + ); + assert_eq!(routed.params.model, "anthropic/claude-3-haiku"); + assert_eq!(routed.raw_params["model"], "anthropic/claude-3-haiku"); + assert_eq!(routed.record()["model_info"], json!({"id": "w", "keep": 1})); + } + + #[rstest] + #[case::no_team_lists_a_team_owned_one(Some("t1"), None, true)] + #[case::no_team_lists_an_unowned_one(None, None, true)] + #[case::the_owner_lists_it(Some("t1"), Some("t1"), true)] + #[case::another_team_does_not(Some("t1"), Some("t2"), false)] + #[case::a_team_lists_an_unowned_one(None, Some("t2"), true)] + fn listed_for_team_lets_no_team_see_every_deployment( + #[case] owner: Option<&str>, + #[case] team: Option<&str>, + #[case] listed: bool, + ) { + let deployment = Deployment::from_spec(spec( + "g", + json!({"model": "openai/a"}), + json!({"id": "a", "team_id": owner}), + )) + .unwrap(); + assert_eq!(deployment.listed_for_team(team), listed); + } + + /// Expected ids computed with `litellm.router_api.deployment_id`. + #[rstest] + #[case::strings( + "g", + json!({"model": "openai/gpt-4o", "api_key": "sk-1"}), + "30256ba697cc98ce2aff32c3b73c4b90514cb199876aef429ffcd91ae826d025" + )] + #[case::scalars( + "claude", + json!({"model": "anthropic/claude-3", "rpm": 10, "timeout": 1.5, "stream": true, "x": null}), + "645fd1504089c1084316573ccfc5dd1dd9622623b169026657484fb64a2c89a2" + )] + #[case::list_and_dict( + "m", + json!({"model": "azure/x", "tags": ["a", "b'c"], "extra_body": {"k": [1, 2.0, "é"], "z": {"q": false}}}), + "64f11e16dabed321268bd92132de2020290ef193c0cf554b2c74faee8badbb96" + )] + fn deployment_id_matches_python( + #[case] group: &str, + #[case] params: Value, + #[case] expected: &str, + ) { + let Value::Object(params) = params else { + unreachable!() + }; + assert_eq!(deployment_id(group, ¶ms), expected); + } + + #[rstest] + #[case::integral(2.0, "2.0")] + #[case::small(0.00001, "1e-05")] + #[case::large(1e16, "1e+16")] + #[case::fraction(0.1, "0.1")] + fn floats_print_as_python_repr(#[case] value: f64, #[case] expected: &str) { + let mut out = String::new(); + python_float(value, &mut out); + assert_eq!(out, expected); + } + + #[rstest] + #[case::group("gpt-4", ResolveKind::Group, "gpt-4")] + #[case::alias("smart", ResolveKind::Alias, "gpt-4")] + #[case::id("dep-b", ResolveKind::Id, "gpt-4")] + #[case::wildcard("anthropic/claude-x", ResolveKind::Wildcard, "anthropic/*")] + fn resolve_follows_python_order( + #[case] name: &str, + #[case] kind: ResolveKind, + #[case] group: &str, + ) { + let registry = Registry::build( + vec![ + spec( + "gpt-4", + json!({"model": "openai/gpt-4"}), + json!({"id": "dep-a"}), + ), + spec( + "gpt-4", + json!({"model": "azure/gpt-4"}), + json!({"id": "dep-b"}), + ), + spec("anthropic/*", json!({"model": "anthropic/*"}), json!({})), + ], + BTreeMap::from([("smart".to_owned(), "gpt-4".to_owned())]), + Vec::new(), + ) + .unwrap(); + let resolution = registry.resolve(name, ResolveScope::default()).unwrap(); + assert_eq!((resolution.kind, resolution.group.as_str()), (kind, group)); + } + + #[rstest] + fn apply_moves_an_edited_deployment_to_the_end() { + let registry = Registry::build( + vec![ + spec("a", json!({"model": "openai/a"}), json!({"id": "1"})), + spec("a", json!({"model": "openai/b"}), json!({"id": "2"})), + ], + BTreeMap::new(), + Vec::new(), + ) + .unwrap(); + let next = registry + .apply( + &ChangeSet { + upserts: vec![spec("a", json!({"model": "openai/c"}), json!({"id": "1"}))], + ..ChangeSet::default() + }, + None, + ) + .unwrap(); + let ids: Vec = next + .group("a") + .unwrap() + .iter() + .map(|d| d.id.clone()) + .collect(); + assert_eq!( + (next.version(), ids), + (1, vec!["2".to_owned(), "1".to_owned()]) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/resolve.rs b/litellm-rust/crates/router-core/src/resolve.rs new file mode 100644 index 00000000000..9075fc36c53 --- /dev/null +++ b/litellm-rust/crates/router-core/src/resolve.rs @@ -0,0 +1,585 @@ +//! A requested name to the deployments that may serve this caller: a port of +//! `_common_checks_available_deployment`. In order: a specific deployment, a deployment id, the +//! alias, a routing group, then for a name no deployment has, a team's public name, every team's +//! public name for a proxy admin, a wildcard; else the group (team-first), kept to the caller's +//! access groups, then deployments whose litellm model is the name, then the first default +//! fallback. Every arm drops strategy markers and deployments that refuse the request's endpoint, +//! and withholds deployments reserved for other teams. + +use std::{collections::BTreeSet, sync::Arc}; + +use jiff::Timestamp; + +use serde_json::Value; + +use crate::{ + access::{filter_by_access_groups, filter_reserved}, + eligibility::eligible_deployments, + error::Error, + host_index::{pattern_to_regex, regex_match}, + registry::{Deployment, DeploymentSpec, Registry, Resolution, ResolveKind, routed_deployment}, + select::is_strategy_marker, +}; + +/// Who is asking, as the request's auth metadata says. +#[derive(Clone, Copy, Debug, Default)] +pub struct Caller<'a> { + /// `user_api_key_team_id`. + pub team_id: Option<&'a str>, + /// The caller's `user_role` is `proxy_admin`. + pub proxy_admin: bool, + /// The caller's `models` plus `team_models`. + pub allowed_models: &'a [String], + /// The endpoint the pick runs under (`endpoint_selection`), which deployments may refuse. + pub endpoint: Option<&'a str>, + /// The model `endpoint_selection` was given: the name an endpoint refusal reads. + pub requested: &'a str, + /// `f"{provider}/{model}"`, the spelling `PatternMatchRouter.get_pattern` tries when no + /// pattern matches the name itself. + pub wildcard_request: Option<&'a str>, +} + +/// `_core_should_include_deployment`: the team's own deployment under its public name, or a +/// deployment named `model_name` the team may use. +fn included(deployment: &Deployment, model_name: &str, team_id: Option<&str>) -> bool { + let info = &deployment.model_info; + if team_id.is_some() + && info.team_id.as_deref() == team_id + && info.team_public_model_name.as_deref() == Some(model_name) + { + return true; + } + deployment.model_name == model_name + && (team_id.is_none() || info.team_id.is_none() || info.team_id.as_deref() == team_id) +} + +/// `_core_get_all_deployments(model_name, team_id)`: the team's deployments under that public +/// name when it has any, else the deployments named `model_name` the team may use; a name the +/// model-name index does not hold is looked for in every deployment, for a team. +pub fn all_deployments( + registry: &Registry, + model_name: &str, + team_id: Option<&str>, +) -> Result>, Error> { + let keep = |deployments: Vec>| -> Vec> { + deployments + .into_iter() + .filter(|d| included(d, model_name, team_id)) + .collect() + }; + // A written index position past the model list raises `IndexError` where it is read. + let out_of_range = || Error::IndexOutOfRange { + model: model_name.to_owned(), + }; + if let Some(team) = team_id { + if registry.team_index_overflows(team, model_name) { + return Err(out_of_range()); + } + if let Some(own) = registry.team_named(team, model_name) { + let own = keep(own); + if !own.is_empty() { + return Ok(own); + } + } + } + if registry.name_index_overflows(model_name) { + return Err(out_of_range()); + } + Ok(match registry.named(model_name) { + Some(named) => keep(named), + None if team_id.is_some() => keep(registry.all().to_vec()), + None => Vec::new(), + }) +} + +/// `_team_deployments_across_teams`: every team's deployments under public name `model`, for a +/// proxy admin calling without a team; a name several teams share is refused. +fn across_teams(registry: &Registry, model: &str) -> Result>, Error> { + if registry.across_teams_index_overflows(model) { + return Err(Error::IndexOutOfRange { + model: model.to_owned(), + }); + } + let deployments = registry.team_named_across_teams(model); + let teams: BTreeSet<&str> = deployments + .iter() + .filter_map(|d| d.model_info.team_id.as_deref()) + .collect(); + if teams.len() > 1 { + return Err(Error::MultipleTeams { + model: model.to_owned(), + }); + } + Ok(deployments) +} + +/// The access groups of the rows `_core_get_model_list(model_name, team_id)` lists: the +/// group's deployments, an alias's target deployments, a routing group's rows (which carry no +/// access groups), and only when there are none of those, the wildcard's deployments. +fn model_access_groups( + registry: &Registry, + model: &str, + team_id: Option<&str>, +) -> BTreeSet { + let mut rows = all_deployments(registry, model, team_id).unwrap_or_default(); + let mut stripped_rows = 0; + if let Some(target) = registry.alias_of(model) { + match registry.routing_group_named(target) { + Some(group) => stripped_rows += routing_group_rows(registry, group, None).len(), + None => rows.extend(all_deployments(registry, target, None).unwrap_or_default()), + } + } + if let Some(group) = registry.routing_group_named(model) { + stripped_rows += routing_group_rows(registry, group, None).len(); + } + if rows.is_empty() && stripped_rows == 0 { + rows = registry + .wildcard_for(model) + .map(|(_, deployments)| deployments) + .unwrap_or_default(); + } + rows.iter() + .flat_map(|d| d.model_info.access_groups.iter().cloned()) + .collect() +} + +fn routing_group_rows( + registry: &Registry, + group: &crate::registry::RoutingGroupSpec, + team_id: Option<&str>, +) -> Vec> { + crate::routing_groups::members(group, |member| { + all_deployments(registry, member, team_id).unwrap_or_default() + }) +} + +/// `_filter_reserved_deployments` for deployments resolved as `model`: markers dropped (only +/// markers is refused), those that refuse the caller's endpoint dropped (`_drop_strategy_markers`), +/// then the deployments an active access window withholds; when that leaves nothing, the window +/// is named in the refusal. +fn reserved( + model: &str, + deployments: Vec>, + caller: Caller<'_>, + now: Timestamp, +) -> Result>, Error> { + let any = !deployments.is_empty(); + let selectable: Vec> = deployments + .into_iter() + .filter(|d| !is_strategy_marker(d)) + .collect(); + if any && selectable.is_empty() { + return Err(Error::OnlyMarkers { + model: model.to_owned(), + }); + } + let selectable = eligible_deployments(model, caller, selectable)?; + let (kept, blocking) = filter_reserved(selectable, caller.team_id, now); + match blocking { + Some(window) if kept.is_empty() => Err(Error::Reserved { + model: model.to_owned(), + until: window.end_hhmm(), + timezone: window.timezone, + }), + _ => Ok(kept), + } +} + +/// `PatternMatchRouter.route`: copies of a pattern's deployments serving `request`. +fn routed_wildcard( + pattern: &str, + request: &str, + deployments: &[Arc], +) -> Vec> { + let groups = regex_match(&pattern_to_regex(pattern), request).unwrap_or_default(); + deployments + .iter() + .map(|deployment| routed_deployment(deployment, &groups, request)) + .collect() +} + +/// `get_pattern` for `model`: the most specific pattern matching it as spelled, else its +/// provider-prefixed spelling, with the pattern's deployments serving that request. +fn wildcard_route( + registry: &Registry, + model: &str, + wildcard_request: Option<&str>, +) -> Option<(String, Vec>)> { + let route = |request: &str| { + registry + .wildcard_for(request) + .map(|(pattern, deployments)| { + let routed = routed_wildcard(&pattern, request, &deployments); + (pattern, routed) + }) + }; + route(model).or_else(|| route(wildcard_request?)) +} + +/// The name resolution would look for under its provider (`get_pattern`'s second spelling), which +/// only the host can infer: one that reaches the pattern routers and that no pattern matches as +/// spelled. +pub(crate) fn wildcard_name<'a>( + registry: &'a Registry, + name: &'a str, + specific_deployment: bool, +) -> Option<&'a str> { + let model = registry.alias_of(name).unwrap_or(name); + (registry.has_wildcards() + && !specific_deployment + && !registry.has_name(model) + && (registry.has_name(name) || registry.deployment(name).is_none()) + && registry.routing_group_named(model).is_none() + && registry.wildcard_for(model).is_none()) + .then_some(model) +} + +/// The first default fallback a name no deployment serves is routed to. +pub fn default_target(dynamic: &crate::config::RouterDynamicConfig) -> Option<&str> { + dynamic + .fallbacks + .default_target() + .or_else(|| dynamic.default_fallbacks.first().map(String::as_str)) +} + +/// The router's `default_deployment`, as resolution serves it. +#[derive(Clone, Copy, Debug)] +pub enum DefaultDeployment<'a> { + Spec(&'a DeploymentSpec), + /// A value Python cannot copy for a name: resolving to it raises. + Raises, +} + +/// The default deployment as served for `model`: a copy whose `litellm_params.model` is the name. +fn default_for(spec: &DeploymentSpec, model: &str) -> Option> { + let mut spec = spec.clone(); + spec.litellm_params + .insert("model".to_owned(), Value::String(model.to_owned())); + Deployment::from_spec(spec).ok().map(Arc::new) +} + +/// `_core_get_routable_deployments`: the deployments resolution would consider for `request`, +/// before any team, cooldown or strategy filter; nothing when it resolves to nothing or refuses. +pub fn routable( + registry: &Registry, + dynamic: &crate::config::RouterDynamicConfig, + request: &crate::host::RouterRequest, +) -> Vec> { + let caller = Caller { + team_id: request.team_id.as_deref(), + proxy_admin: request.proxy_admin, + allowed_models: &request.allowed_models, + endpoint: request.endpoint.as_deref(), + requested: &request.model, + wildcard_request: request.wildcard_request.as_deref(), + }; + match resolve_request( + registry, + &request.model, + false, + caller, + default_target(dynamic), + dynamic.default_ref(), + Timestamp::now(), + ) { + Ok(Some(resolution)) => resolution.deployments, + Ok(None) | Err(_) => Vec::new(), + } +} + +/// Resolve `name` for `caller`. `None` when nothing serves it (`no_healthy_deployments`). +pub fn resolve_request( + registry: &Registry, + name: &str, + specific_deployment: bool, + caller: Caller<'_>, + default_target: Option<&str>, + default_deployment: Option>, + now: Timestamp, +) -> Result, Error> { + let team_id = caller.team_id; + let resolved = |kind, group: &str, deployments| { + Ok(Some(Resolution { + kind, + group: group.to_owned(), + deployments, + })) + }; + if specific_deployment { + let matching = registry + .all() + .iter() + .filter(|d| d.params.model == name) + .cloned() + .collect(); + let kept = reserved(name, matching, caller, now)?; + return resolved(ResolveKind::SpecificDeployment, name, kept); + } + if !registry.has_name(name) + && let Some(deployment) = registry.deployment(name) + { + let group = deployment.model_name.clone(); + let litellm_model = deployment.params.model.clone(); + let kept = reserved(&litellm_model, vec![deployment], caller, now)?; + return resolved(ResolveKind::Id, &group, kept); + } + let aliased = registry.alias_of(name); + let model = aliased.unwrap_or(name); + let routing_group = registry + .routing_group_named(model) + .map(|group| routing_group_rows(registry, group, team_id)); + if routing_group.is_none() && !registry.has_name(model) { + let early = match team_id { + Some(team) => Some(all_deployments(registry, model, Some(team))?), + None if caller.proxy_admin => Some(across_teams(registry, model)?), + None => None, + } + .filter(|deployments| !deployments.is_empty()); + if let Some(deployments) = early { + let kept = reserved(model, deployments, caller, now)?; + return resolved(ResolveKind::Group, model, kept); + } + // `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one. + let team_owned = team_id + .map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team)) + .filter(|deployments| !deployments.is_empty()); + if let Some(deployments) = team_owned { + let routed = deployments.iter().flat_map(|deployment| { + let public = deployment.model_info.team_public_model_name.as_deref(); + let pattern = public.unwrap_or(model); + routed_wildcard(pattern, model, std::slice::from_ref(deployment)) + }); + let kept = reserved(model, routed.collect(), caller, now)?; + return resolved(ResolveKind::Wildcard, model, kept); + } + if let Some((pattern, deployments)) = + wildcard_route(registry, model, caller.wildcard_request) + { + let kept = reserved(model, deployments, caller, now)?; + return resolved(ResolveKind::Wildcard, &pattern, kept); + } + match default_deployment { + Some(DefaultDeployment::Raises) => { + return Err(Error::DefaultDeploymentRaises { + model: model.to_owned(), + }); + } + Some(DefaultDeployment::Spec(spec)) => { + if let Some(deployment) = default_for(spec, model) { + let kept = reserved(model, vec![deployment], caller, now)?; + return resolved(ResolveKind::Default, model, kept); + } + } + None => {} + } + } + let (kind, listed) = match routing_group { + Some(deployments) => (ResolveKind::RoutingGroup, deployments), + None if aliased.is_some() => ( + ResolveKind::Alias, + all_deployments(registry, model, team_id)?, + ), + None => ( + ResolveKind::Group, + all_deployments(registry, model, team_id)?, + ), + }; + let narrow = |model: &str, deployments: Vec>| { + let allowed = filter_by_access_groups(model, deployments, caller.allowed_models, || { + model_access_groups(registry, model, team_id) + }); + reserved(model, allowed, caller, now) + }; + let listed_any = !listed.is_empty(); + let deployments = narrow(model, listed)?; + if !deployments.is_empty() { + return resolved(kind, model, deployments); + } + let mut access_emptied = listed_any; + if !listed_any { + // The caller may have named a deployment's litellm model instead. + let by_model: Vec> = registry + .all() + .iter() + .filter(|d| d.params.model == model) + .cloned() + .collect(); + let found = !by_model.is_empty(); + let deployments = narrow(model, by_model)?; + if !deployments.is_empty() { + return resolved(ResolveKind::Group, model, deployments); + } + access_emptied = found; + } + if let Some(target) = default_target.filter(|target| !access_emptied && !target.is_empty()) { + let deployments = narrow(target, all_deployments(registry, target, team_id)?)?; + if !deployments.is_empty() { + return resolved(ResolveKind::Group, target, deployments); + } + // `_common_checks_available_deployment` re-assigned the name: its error names the target. + return Err(Error::UnknownModel { + model: target.to_owned(), + }); + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use serde_json::{Value, json}; + + use super::*; + use crate::registry::DeploymentSpec; + + fn registry(entries: Value) -> Registry { + let specs: Vec = serde_json::from_value(entries).unwrap(); + Registry::build(specs, BTreeMap::new(), Vec::new()).unwrap() + } + + fn ids(resolution: Option) -> Vec { + resolution + .map(|r| r.deployments.iter().map(|d| d.id.clone()).collect()) + .unwrap_or_default() + } + + #[test] + fn an_access_group_only_caller_is_kept_to_its_groups() { + let registry = registry(json!([ + {"model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a", "access_groups": ["A"]}}, + {"model_name": "g", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "b", "access_groups": ["B"]}}, + ])); + let allowed = vec!["A".to_owned()]; + let caller = Caller { + allowed_models: &allowed, + ..Caller::default() + }; + let resolved = resolve_request(®istry, "g", false, caller, None, None, Timestamp::now()); + assert_eq!(ids(resolved.unwrap()), ["a"]); + // The litellm model string is not narrowed (python-fixes-to-make.md). + let by_model = resolve_request( + ®istry, + "openai/b", + false, + caller, + None, + None, + Timestamp::now(), + ); + assert_eq!(ids(by_model.unwrap()), ["b"]); + } + + #[test] + fn a_proxy_admin_is_refused_a_public_name_several_teams_share() { + let registry = registry(json!([ + {"model_name": "x", "litellm_params": {"model": "openai/x"}, "model_info": {"id": "x", "team_id": "tx", "team_public_model_name": "pub"}}, + {"model_name": "y", "litellm_params": {"model": "openai/y"}, "model_info": {"id": "y", "team_id": "ty", "team_public_model_name": "pub"}}, + ])); + let caller = Caller { + proxy_admin: true, + ..Caller::default() + }; + let refused = resolve_request( + ®istry, + "pub", + false, + caller, + None, + None, + Timestamp::now(), + ); + assert!(matches!(refused, Err(Error::MultipleTeams { model }) if model == "pub")); + } + + fn wildcard_group() -> Value { + json!([{ + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "w", "supports_prompt_cache_breakpoint": true}, + }]) + } + + #[test] + fn a_wildcard_serves_the_matched_model_without_its_cache_flags() { + let registry = registry(wildcard_group()); + let resolved = resolve_request( + ®istry, + "openai/gpt-4o", + false, + Caller::default(), + None, + None, + Timestamp::now(), + ); + let resolution = resolved.unwrap().unwrap(); + assert_eq!(resolution.kind, ResolveKind::Wildcard); + assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o"); + assert_eq!( + resolution.deployments[0].record()["model_info"], + json!({"id": "w"}) + ); + } + + #[test] + fn an_alias_to_a_wildcard_fills_the_model_from_its_target() { + let specs: Vec = serde_json::from_value(wildcard_group()).unwrap(); + let aliases = BTreeMap::from([("al".to_owned(), "openai/gpt-4o".to_owned())]); + let registry = Registry::build(specs, aliases, Vec::new()).unwrap(); + let resolved = resolve_request( + ®istry, + "al", + false, + Caller::default(), + None, + None, + Timestamp::now(), + ); + assert_eq!( + resolved.unwrap().unwrap().deployments[0].params.model, + "openai/gpt-4o" + ); + } + + #[test] + fn a_name_no_pattern_matches_is_tried_under_its_provider() { + let registry = registry(wildcard_group()); + assert_eq!(wildcard_name(®istry, "gpt-4o", false), Some("gpt-4o")); + assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false), None); + let caller = Caller { + wildcard_request: Some("openai/gpt-4o"), + ..Caller::default() + }; + let resolved = resolve_request( + ®istry, + "gpt-4o", + false, + caller, + None, + None, + Timestamp::now(), + ); + let resolution = resolved.unwrap().unwrap(); + assert_eq!(resolution.group, "openai/*"); + assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o"); + } + + #[test] + fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() { + let registry = registry(json!([ + {"model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": { + "id": "a", "enforce_supported_endpoints": true, "supported_endpoints": ["/v1/embeddings"], + }}, + ])); + let caller = Caller { + endpoint: Some("/v1/chat/completions"), + requested: "a", + ..Caller::default() + }; + let refused = resolve_request(®istry, "a", false, caller, None, None, Timestamp::now()); + assert!(matches!( + refused, + Err(Error::EndpointNotSupported { model, .. }) if model == "openai/a" + )); + } +} diff --git a/litellm-rust/crates/router-core/src/retry.rs b/litellm-rust/crates/router-core/src/retry.rs new file mode 100644 index 00000000000..3de0ad601e3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/retry.rs @@ -0,0 +1,219 @@ +//! Retry decisions: how many retries a `RetryPolicy` grants for a failure, and whether a failure +//! is retried in the same group at all. Mirrors `get_num_retries_from_retry_policy` and +//! `resolve_retry_policy` in `litellm_core_utils/get_retry_from_policy.py`, and +//! `Router.should_retry_this_error`. + +use std::collections::BTreeMap; + +use crate::{ + config::RetryPolicy, + error::{AttemptError, FailureKind}, + health::should_retry_status, +}; + +/// `resolve_retry_policy`: the group's own policy when `model_group_retry_policy` has one, else +/// the router's `retry_policy`. +pub fn resolve_retry_policy<'a>( + retry_policy: Option<&'a RetryPolicy>, + model_group: &str, + model_group_retry_policy: Option<&'a BTreeMap>, +) -> Option<&'a RetryPolicy> { + model_group_retry_policy + .and_then(|policies| policies.get(model_group)) + .or(retry_policy) +} + +impl RetryPolicy { + /// `get_num_retries_from_retry_policy`: `DefaultRetries` for an `UnsupportedEndpointError`; + /// else `NotFoundErrorRetries` for any 404 answer, then the most specific exception class + /// the policy names, then `DefaultRetries`. + pub fn retries_for(&self, failure: &AttemptError) -> Option { + if failure.is_a("UnsupportedEndpointError") { + return self.DefaultRetries; + } + let not_found = (failure.status == Some(404)) + .then_some(self.NotFoundErrorRetries) + .flatten(); + not_found + .or_else(|| { + failure + .class_names() + .iter() + .find_map(|class| self.field_for(class)) + }) + .or(self.DefaultRetries) + } + + /// `_RETRIES_BY_EXCEPTION_TYPE`: the policy field an exception class reads. + fn field_for(&self, class: &str) -> Option { + match class { + "AuthenticationError" => self.AuthenticationErrorRetries, + "Timeout" => self.TimeoutErrorRetries, + "RateLimitError" => self.RateLimitErrorRetries, + "ContentPolicyViolationError" => self.ContentPolicyViolationErrorRetries, + "BadRequestError" => self.BadRequestErrorRetries, + "ServiceUnavailableError" => self.ServiceUnavailableErrorRetries, + "InternalServerError" => self.InternalServerErrorRetries, + _ => None, + } + } +} + +/// What `should_retry_this_error` knows besides the failure. +#[derive(Clone, Copy, Debug, Default)] +pub struct RetryContext { + pub healthy: usize, + pub all: usize, + /// `context_window_fallbacks is not None`. + pub has_context_window_fallbacks: bool, + /// `content_policy_fallbacks is not None`. + pub has_content_policy_fallbacks: bool, + /// `regular_fallbacks is not None and len(regular_fallbacks) > 0`. + pub has_regular_fallbacks: bool, +} + +/// `should_retry_this_error`, as a predicate: `false` where Python raises the error. +pub fn should_retry(failure: &AttemptError, ctx: RetryContext) -> bool { + if failure.is_a("ContextWindowExceededError") && ctx.has_context_window_fallbacks { + return false; + } + if failure.is_a("ContentPolicyViolationError") && ctx.has_content_policy_fallbacks { + return false; + } + if let Some(status) = failure.status + && !should_retry_status(status) + && !matches!(status, 401 | 403) + && !failure.is_a("UnsupportedEndpointError") + { + return false; + } + if failure.is_a("NotFoundError") { + return false; + } + if failure.is_a("RateLimitError") && ctx.healthy == 0 && ctx.has_regular_fallbacks { + return false; + } + if (failure.is_a("AuthenticationError") || failure.is_a("PermissionDeniedError")) + && ctx.all <= 1 + { + return false; + } + ctx.healthy > 0 +} + +/// `ContentPolicyViolationError`'s classes, most specific first. +pub(crate) fn content_policy_classes() -> Vec> { + [ + "ContentPolicyViolationError", + "BadRequestError", + "APIStatusError", + "APIError", + "OpenAIError", + "Exception", + "BaseException", + "object", + ] + .into_iter() + .map(Into::into) + .collect() +} + +/// The exception classes a failure kind stands for, most specific first, for a host that +/// reports no class names (`litellm/exceptions.py`). +pub(crate) fn kind_classes(kind: FailureKind) -> &'static [&'static str] { + match kind { + FailureKind::RateLimit => &["RateLimitError"], + FailureKind::ContextWindowExceeded => &["ContextWindowExceededError", "BadRequestError"], + FailureKind::ContentPolicyViolation => &["ContentPolicyViolationError", "BadRequestError"], + FailureKind::Authentication => &["AuthenticationError"], + FailureKind::PermissionDenied => &["PermissionDeniedError"], + FailureKind::NotFound => &["NotFoundError"], + FailureKind::BadRequest => &["BadRequestError"], + FailureKind::Timeout => &["Timeout", "APITimeoutError", "APIConnectionError"], + FailureKind::ApiConnection => &["APIConnectionError"], + FailureKind::InternalServer => &["InternalServerError"], + FailureKind::ServiceUnavailable => &["ServiceUnavailableError"], + FailureKind::BadGateway => &["BadGatewayError"], + FailureKind::CallerCredentialAuthentication => { + &["CallerCredentialAuthenticationError", "AuthenticationError"] + } + FailureKind::CallerCredentialRateLimit => { + &["CallerCredentialRateLimitError", "RateLimitError"] + } + FailureKind::GuardrailIntervention | FailureKind::Declined | FailureKind::Other => &[], + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn failure(kind: FailureKind, status: Option, classes: &[&str]) -> AttemptError { + let mut failure = AttemptError::new(kind, status); + failure.classes = classes.iter().map(|class| (*class).into()).collect(); + failure + } + + #[rstest] + #[case::own_field(failure(FailureKind::InternalServer, Some(500), &[]), Some(4))] + #[case::default(failure(FailureKind::BadGateway, Some(502), &[]), Some(7))] + #[case::not_found_status(failure(FailureKind::Other, Some(404), &["APIError"]), Some(1))] + #[case::unsupported_endpoint( + failure(FailureKind::Declined, Some(400), &["UnsupportedEndpointError", "BadRequestError"]), + Some(7) + )] + #[case::malformed_reads_bad_request( + failure(FailureKind::Declined, Some(400), &["MalformedRequestFieldError", "BadRequestError"]), + Some(3) + )] + fn retries_follow_the_most_specific_class( + #[case] failure: AttemptError, + #[case] expected: Option, + ) { + let policy = RetryPolicy { + InternalServerErrorRetries: Some(4), + BadRequestErrorRetries: Some(3), + NotFoundErrorRetries: Some(1), + DefaultRetries: Some(7), + ..RetryPolicy::default() + }; + assert_eq!(policy.retries_for(&failure), expected); + } + + #[rstest] + #[case::rate_limit_with_healthy(FailureKind::RateLimit, Some(429), 1, 2, false, true)] + #[case::rate_limit_none_healthy_with_fallbacks( + FailureKind::RateLimit, + Some(429), + 0, + 2, + true, + false + )] + #[case::bad_request(FailureKind::BadRequest, Some(400), 2, 2, false, false)] + #[case::auth_lone_deployment(FailureKind::Authentication, Some(401), 1, 1, false, false)] + #[case::auth_with_siblings(FailureKind::Authentication, Some(401), 1, 2, false, true)] + #[case::not_found(FailureKind::NotFound, Some(404), 2, 2, false, false)] + #[case::server_error_no_healthy(FailureKind::InternalServer, Some(500), 0, 2, false, false)] + fn should_retry_follows_python( + #[case] kind: FailureKind, + #[case] status: Option, + #[case] healthy: usize, + #[case] all: usize, + #[case] has_regular_fallbacks: bool, + #[case] expected: bool, + ) { + let ctx = RetryContext { + healthy, + all, + has_regular_fallbacks, + ..RetryContext::default() + }; + assert_eq!( + should_retry(&AttemptError::new(kind, status), ctx), + expected + ); + } +} diff --git a/litellm-rust/crates/router-core/src/routing_groups.rs b/litellm-rust/crates/router-core/src/routing_groups.rs new file mode 100644 index 00000000000..fcf6c2febf3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/routing_groups.rs @@ -0,0 +1,48 @@ +//! Routing groups: a callable name over the deployments of its member model groups +//! (`_get_routing_group_deployments`). A priority group serves each member's priority as the +//! member deployments' `order` (`apply_routing_group_priority`), so the order filter and order +//! fallbacks walk the priorities. + +use std::sync::Arc; + +use serde_json::Value; + +use crate::registry::{Deployment, RoutingGroupSpec}; + +/// The member deployments of `group`, in member order, with a priority group's priorities as +/// their `order`. +pub fn members( + group: &RoutingGroupSpec, + deployments_of: impl Fn(&str) -> Vec>, +) -> Vec> { + let priorities = group + .model_priorities + .as_ref() + .filter(|_| is_priority(group)); + group + .models + .iter() + .flat_map(|member| { + let priority = priorities.and_then(|p| p.get(member)).copied(); + deployments_of(member) + .into_iter() + .map(move |deployment| match priority { + Some(order) => with_order(&deployment, order), + None => deployment, + }) + }) + .collect() +} + +/// Whether `group` is a priority group. +pub fn is_priority(group: &RoutingGroupSpec) -> bool { + matches!(&group.routing_strategy, crate::config::RoutingStrategy::Other(name) if name == "priority") +} + +fn with_order(deployment: &Arc, order: i64) -> Arc { + let mut prioritized = Deployment::clone(deployment); + prioritized + .raw_params + .insert("order".to_owned(), Value::from(order)); + Arc::new(prioritized) +} diff --git a/litellm-rust/crates/router-core/src/routing_plugins.rs b/litellm-rust/crates/router-core/src/routing_plugins.rs new file mode 100644 index 00000000000..236de6fc521 --- /dev/null +++ b/litellm-rust/crates/router-core/src/routing_plugins.rs @@ -0,0 +1,114 @@ +//! `Router(plugins=[...])`: the routing-plugin pipeline. The host runs the plugins (live Python +//! objects) in the async pick's pre-routing hook and reports the candidate models they left on +//! the request (`_routing_plugin_candidate_models`); router-core decides which deployments that +//! keeps (`_filter_by_routing_plugin_candidates`) and refuses the paths that never run the +//! pipeline (`_get_available_deployment_unresolved`). + +use std::sync::Arc; + +use crate::{config::RoutingStrategy, error::Error, host::RouterRequest, registry::Deployment}; + +/// Whether the router's own strategy has an async selector: the others pick on the sync path +/// even for async callers (`_async_get_available_deployment`). +pub(crate) fn picks_async(strategy: &RoutingStrategy) -> bool { + matches!( + strategy, + RoutingStrategy::UsageBasedV2 + | RoutingStrategy::SimpleShuffle + | RoutingStrategy::CostBased + | RoutingStrategy::LatencyBased + | RoutingStrategy::LeastBusy + ) +} + +/// The sync pick never runs the pipeline, so a router with plugins refuses it rather than +/// letting a policy plugin be bypassed. `request` is a pick or a call, never a listing. +pub(crate) fn refuse_sync_pick( + has_plugins: bool, + strategy: &RoutingStrategy, + request: &RouterRequest, +) -> Result<(), Error> { + if has_plugins + && request.pass_through.is_none() + && (!request.pre_routes || !picks_async(strategy)) + { + return Err(Error::RoutingPluginsNeedAsync); + } + Ok(()) +} + +/// `_filter_by_routing_plugin_candidates`: keep the deployments whose litellm model the plugins +/// left; nothing left is an error, never a fallback to the unfiltered pool. +pub(crate) fn narrow( + healthy: Vec>, + models: Option<&[String]>, +) -> Result>, Error> { + let Some(models) = models else { + return Ok(healthy); + }; + let kept: Vec> = healthy + .into_iter() + .filter(|d| models.contains(&d.params.model)) + .collect(); + if kept.is_empty() { + return Err(Error::RoutingPluginsLeftNothing { + models: models.to_vec(), + }); + } + Ok(kept) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn deployment(id: &str, model: &str) -> Arc { + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": {"model": model}, "model_info": {"id": id}, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + #[rstest] + fn the_plugins_models_narrow_the_pool() { + let pool = vec![deployment("a", "openai/a"), deployment("b", "openai/b")]; + let kept = narrow(pool, Some(&["openai/b".to_owned()])).unwrap(); + assert_eq!( + kept.iter().map(|d| d.id.as_str()).collect::>(), + ["b"] + ); + } + + #[rstest] + fn narrowing_to_nothing_is_an_error() { + let pool = vec![deployment("a", "openai/a")]; + assert!(matches!( + narrow(pool, Some(&[])), + Err(Error::RoutingPluginsLeftNothing { models }) if models.is_empty() + )); + } + + #[rstest] + #[case::async_pick(true, RoutingStrategy::SimpleShuffle, false)] + #[case::sync_pick(false, RoutingStrategy::SimpleShuffle, true)] + #[case::v1_strategy(true, RoutingStrategy::UsageBased, true)] + fn only_the_async_pick_runs_plugins( + #[case] pre_routes: bool, + #[case] strategy: RoutingStrategy, + #[case] refused: bool, + ) { + let request = RouterRequest { + pre_routes, + ..RouterRequest::default() + }; + assert_eq!( + refuse_sync_pick(true, &strategy, &request).is_err(), + refused + ); + } +} diff --git a/litellm-rust/crates/router-core/src/select.rs b/litellm-rust/crates/router-core/src/select.rs new file mode 100644 index 00000000000..fefa7f25ad1 --- /dev/null +++ b/litellm-rust/crates/router-core/src/select.rs @@ -0,0 +1,795 @@ +//! Selection: resolve a name, filter its deployments in the order of +//! `Router._core_async_get_healthy_deployments` (team, web search, cooldown, blocked, plugins, +//! pre-call checks, tags, order, retry skips), then pick one with the strategy. Simple-shuffle is +//! native (`router_strategy/simple_shuffle.py`); other strategies go through `Host::select`. + +use std::{collections::BTreeMap, sync::Arc}; + +use parking_lot::Mutex; +use rand::{Rng, SeedableRng, rngs::StdRng}; + +use crate::{ + Core, RouterStateSnapshot, + config::{Op, RoutingStrategy}, + error::Error, + health_routing::WhenAllUnhealthy, + host::{RouterRequest, RoutingContext, SelectStage}, + registry::{Deployment, Registry, ResolveKind}, + resolve::{Caller, resolve_request}, + strategy::{StrategyScope, routing_context}, +}; + +/// Randomness for the shuffle and retry jitter, injectable so tests are deterministic. +pub trait Randomness: Send + Sync { + /// A float in `[0, 1)`, as Python's `random.random()`. + fn next_f64(&self) -> f64; + + /// An index below `len`, as Python's `random.choice` draws one. + fn index_below(&self, len: usize) -> usize { + ((self.next_f64() * len as f64) as usize).min(len - 1) + } +} + +pub struct ThreadRandomness; + +impl Randomness for ThreadRandomness { + fn next_f64(&self) -> f64 { + rand::thread_rng().r#gen() + } +} + +pub struct SeededRandomness(Mutex); + +impl SeededRandomness { + pub fn new(seed: u64) -> Self { + Self(Mutex::new(StdRng::seed_from_u64(seed))) + } +} + +impl Randomness for SeededRandomness { + fn next_f64(&self) -> f64 { + self.0.lock().r#gen() + } +} + +/// Which pass-through pick a request makes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PassThrough { + /// `get_available_deployment_for_pass_through` (and `select(kind="pass_through")`): the + /// pass-through deployments are chosen first, then cooldown and blocked filters, and only a + /// priority group filters by order. + Listed, + /// `async_get_available_deployment_for_pass_through`: the full healthy chain, then the + /// pass-through filter. + Healthy, +} + +/// A group's deployments before and after filtering. +#[derive(Clone, Debug)] +pub struct Candidates { + /// The name that was resolved (Python's `model`). + pub name: String, + pub model_group: String, + pub kind: ResolveKind, + /// Every deployment the caller may use (after the team filter). + pub all: Vec>, + pub healthy: Vec>, + /// The provider-prefixed spelling the name resolved under (`get_pattern`), when the host + /// was asked for one. + pub wildcard_request: Option, +} + +impl Candidates { + /// The `model` a no-deployments error names: what `_common_checks_available_deployment` + /// returned, so the alias-resolved name a wildcard serves and not its pattern. + pub(crate) fn failed_model(&self, registry: &Registry) -> String { + let model = match self.kind { + ResolveKind::Wildcard => registry.alias_of(&self.name).unwrap_or(&self.name), + _ => self.model_group.as_str(), + }; + model.to_owned() + } +} + +/// Per-attempt selection inputs the call loop carries. +#[derive(Clone, Debug, Default)] +pub(crate) struct SelectInputs { + pub target_order: Option, + pub skipped_ids: Vec, + /// A weighted failover hop's `_excluded_deployment_ids`: never picked. + pub excluded_ids: Arc<[String]>, + /// Stop before the plugins, order and skip filters: the group less cooldowns and blocked + /// deployments, which `_async_get_healthy_deployments` counts when deciding whether to retry. + pub count_only: bool, + /// The level's name, which an endpoint refusal reads (`endpoint_selection(model=...)`): the + /// name before pre-routing replaced it. + pub requested: Option, + /// A fallback hop: `run_async_fallback` pops the request's `_target_order`, so only the + /// hop's own order applies. + pub fallback_hop: bool, +} + +/// `simple_shuffle`: weighted by `_router_weights[group]`, else `weight`, else `rpm`, else +/// `tpm`, each normalized by its maximum, else uniform. A metric is read only when every set +/// before it weighed nothing, and a non-numeric one fails the pick then (`_metric_weight`). +pub fn simple_shuffle( + deployments: &[Arc], + model_group: &str, + router_weights: Option<&BTreeMap>>, + rng: &dyn Randomness, +) -> Result>, Error> { + if deployments.is_empty() { + return Ok(None); + } + let scoped = router_weights + .and_then(|weights| weights.get(model_group)) + .filter(|weights| !weights.is_empty()) + .map(|weights| { + deployments + .iter() + .map(|d| weights.get(&d.id).copied().unwrap_or(0.0)) + .collect::>() + }); + let metric_sets = ["weight", "rpm", "tpm"].into_iter().map(|metric| { + deployments + .iter() + .map(|d| d.metric_weight(metric)) + .collect::>>() + .ok_or(Error::NonNumericWeight { metric }) + }); + for weights in scoped.into_iter().map(Ok).chain(metric_sets) { + let weights = weights?; + let largest = weights.iter().copied().fold(0.0, f64::max); + if largest <= 0.0 { + continue; + } + let normalized: Vec = weights.iter().map(|weight| weight / largest).collect(); + let total: f64 = normalized.iter().sum(); + if total <= 0.0 { + continue; + } + // `random.choices`: bisect_right over the cumulative weights. + let target = rng.next_f64() * total; + let mut cumulative = 0.0; + let index = normalized + .iter() + .position(|weight| { + cumulative += weight; + target < cumulative + }) + .unwrap_or(deployments.len() - 1); + return Ok(Some(deployments[index].clone())); + } + Ok(Some( + deployments[rng.index_below(deployments.len())].clone(), + )) +} + +/// A pre-routing strategy marker (`classify_strategy_router_model`): its model is `auto_router/...`. +pub fn is_strategy_marker(deployment: &Deployment) -> bool { + deployment.params.model.starts_with("auto_router/") +} + +/// `litellm_params.use_in_pass_through`, read for truthiness as Python does. +fn uses_pass_through(deployment: &Deployment) -> bool { + match deployment.raw_params.get("use_in_pass_through") { + None | Some(serde_json::Value::Null) => false, + Some(serde_json::Value::Bool(value)) => *value, + Some(serde_json::Value::Number(value)) => value.as_f64().is_some_and(|n| n != 0.0), + Some(serde_json::Value::String(value)) => !value.is_empty(), + Some(serde_json::Value::Array(value)) => !value.is_empty(), + Some(serde_json::Value::Object(value)) => !value.is_empty(), + } +} + +/// `filter_team_based_models`: a team-owned deployment serves only its team, except that a proxy +/// admin without a team keeps deployments that are all team-owned and listed under the model the +/// request names, unless they belong to several teams. +fn filter_team_based( + deployments: Vec>, + request: &RouterRequest, +) -> Result>, Error> { + let team_id = request.team_id.as_deref(); + if team_id.is_none() + && request.proxy_admin + && let Some(requested) = request.admin_model.as_deref() + && !deployments.is_empty() + && deployments.iter().all(|d| { + d.model_info.team_id.is_some() + && (d.model_name == requested + || d.model_info.team_public_model_name.as_deref() == Some(requested)) + }) + { + let teams: std::collections::BTreeSet<&str> = deployments + .iter() + .filter_map(|d| d.model_info.team_id.as_deref()) + .collect(); + if teams.len() > 1 { + return Err(Error::MultipleTeams { + model: requested.to_owned(), + }); + } + return Ok(deployments); + } + Ok(deployments + .into_iter() + .filter(|d| d.usable_by_team(team_id)) + .collect()) +} + +impl Core { + fn context<'a>( + &'a self, + snapshot: &'a RouterStateSnapshot, + op: &'a Op, + request: &'a RouterRequest, + model_group: &'a str, + name: &'a str, + scope: StrategyScope, + ) -> RoutingContext<'a> { + RoutingContext { + registry: &snapshot.registry, + settings: &snapshot.settings, + health: &self.health, + request, + op, + model_group, + name, + scope, + pool: &[], + excluded_ids: &[], + skipped_ids: &[], + target_order: None, + } + } + + /// The eligible deployments for `name`, in filter-chain order. + pub(crate) async fn candidates( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &RouterRequest, + name: &str, + inputs: &SelectInputs, + ) -> Result { + let registry = &snapshot.registry; + // The async pick's pre-routing hook runs the routing plugins before anything resolves; + // every pick but a listed pass-through one reads what they left on the request. + let plugin_models = if snapshot.settings.routing_plugins + && !inputs.count_only + && request.pass_through != Some(PassThrough::Listed) + { + let ctx = self.context(snapshot, op, request, name, name, StrategyScope::Default); + self.host + .routing_plugin_models(&ctx, request.pre_routes) + .await? + } else { + None + }; + // No deployment for the name: route to the first default fallback. + let default_target = crate::resolve::default_target(&snapshot.settings.dynamic); + // `get_pattern`: a name no pattern matches is tried again under its inferred provider. + let wildcard_name = + crate::resolve::wildcard_name(registry, name, request.specific_deployment); + let wildcard_request = match wildcard_name { + Some(model) => { + let scope = StrategyScope::Default; + let ctx = self.context(snapshot, op, request, model, model, scope); + self.host.provider_request(&ctx).await? + } + None => None, + }; + // `endpoint_selection` wraps the attempt, not the count a retry decision reads. + let endpoint = request.endpoint.as_deref().filter(|_| !inputs.count_only); + let caller = Caller { + team_id: request.team_id.as_deref(), + proxy_admin: request.proxy_admin, + allowed_models: &request.allowed_models, + endpoint, + requested: inputs.requested.as_deref().unwrap_or(name), + wildcard_request: wildcard_request.as_deref(), + }; + let resolution = resolve_request( + registry, + name, + request.specific_deployment, + caller, + default_target, + snapshot.settings.dynamic.default_ref(), + jiff::Timestamp::now(), + )? + .ok_or_else(|| Error::UnknownModel { + // The name `_common_checks_available_deployment` looked for, past the alias. + model: registry.alias_of(name).unwrap_or(name).to_owned(), + })?; + let first_model = resolution + .deployments + .first() + .map(|d| d.params.model.clone()); + let team_id = request.team_id.as_deref(); + let usable = filter_team_based(resolution.deployments.clone(), request)?; + // `_filter_reserved_deployments`: PTU shares keep holders to their share and everyone + // else to the deployments split into none. + let (mut all, withheld) = crate::ptu::filter_shared(usable, team_id); + if withheld && all.is_empty() { + return Err(Error::PtuReserved { + model: match resolution.kind { + ResolveKind::Id => first_model.unwrap_or_else(|| name.to_owned()), + ResolveKind::Wildcard | ResolveKind::SpecificDeployment => name.to_owned(), + _ => resolution.group.clone(), + }, + }); + } + // `filter_web_search_deployments` runs before `request_deployments` is read, so every later + // stage sees the group without the deployments it dropped. + if request.web_search && !inputs.count_only && !resolution.kind.is_single() { + all.retain(|d| d.supports_web_search()); + } + // `_without_deployments_that_cannot_serve_route`, right after the web search filter: the + // deployments whose provider cannot serve the call, unless that leaves nothing. Only a pick + // runs it (`_route_capability_pin`): the retry count reads the whole group. + if let Some(handler) = request.route_handler.as_deref() + && !inputs.count_only + && !resolution.kind.is_single() + && !all.is_empty() + { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + let stage = SelectStage::RouteIncapable { + handler: handler.to_owned(), + }; + let incapable = self.host.select(&stage, &ctx, all.clone()).await?; + let capable: Vec> = all + .iter() + .filter(|d| !incapable.iter().any(|i| i.id == d.id)) + .cloned() + .collect(); + if !capable.is_empty() { + all = capable; + } + } + let listed = + request.pass_through == Some(PassThrough::Listed) && !resolution.kind.is_single(); + if listed { + all.retain(|d| uses_pass_through(d)); + if all.is_empty() { + // The name `_common_checks_available_deployment` resolved: an alias's target, a + // routing group's name, the alias-resolved name a wildcard serves. + let model = match resolution.kind { + ResolveKind::Wildcard => registry.alias_of(name).unwrap_or(name), + ResolveKind::SpecificDeployment => name, + _ => resolution.group.as_str(), + }; + return Err(Error::NoPassThroughDeployments { + model: model.to_owned(), + }); + } + } + + // A deployment named by its id is served even while it cools down + // (`get_deployment_by_model_id` returns before the cooldown filter). + let settings = &snapshot.settings; + let cooling = if resolution.kind.is_single() { + Vec::new() + } else { + // `async_get_cooldown_deployments` reads every deployment's, as one throttled batch. + self.health + .active_cooldowns(&snapshot.registry.ids()) + .await? + }; + // Health-check filtering runs before the cooldown filter, except for a deployment named + // by its id and the healthy count a retry decision reads. + let checked = if resolution.kind.is_single() || inputs.count_only { + all.clone() + } else { + let when_all_unhealthy = if request.health_check_probe { + WhenAllUnhealthy::Nothing + } else { + WhenAllUnhealthy::KeepAll + }; + self.health + .health_check_filter(settings, all.clone(), when_all_unhealthy) + .await + }; + let mut healthy: Vec> = checked + .iter() + .filter(|d| !cooling.iter().any(|(id, _)| *id == d.id)) + .cloned() + .collect(); + // The safety net when health-check routing drives cooldowns (an allowed_fails_policy): + // never on a listed pass-through pick or the healthy count. + if healthy.is_empty() + && !listed + && !inputs.count_only + && settings.enable_health_check_routing + && settings.allowed_fails_policy.is_some() + { + healthy = checked; + } + if resolution.kind.is_single() + && let Some(deployment) = all.first() + { + // `_core_async_get_healthy_deployments`: a deployment named by id is checked alone. + if deployment.model_info.blocked { + return Err(Error::Paused { + model: deployment.params.model.clone(), + }); + } + if request.pass_through.is_some() && !uses_pass_through(deployment) { + return Err(Error::NotPassThrough { + id: deployment.id.clone(), + }); + } + } + healthy.retain(|d| !d.model_info.blocked); + // `continuation_deployments`: a mid-stream continuation never lands on a deployment the host + // named as unable to take its prefill. + if let Some(excluded) = request + .body + .get("continuation_excluded_ids") + .and_then(serde_json::Value::as_array) + { + healthy.retain(|d| !excluded.iter().any(|id| id.as_str() == Some(d.id.as_str()))); + } + let request_order = request.target_order.filter(|_| !inputs.fallback_hop); + // `async_callback_filter_deployments` runs after the cooldown and blocked filters on every + // async candidate list but a deployment named by id and a listed pass-through pick. + if settings.callback_filters + && !inputs.count_only + && !resolution.kind.is_single() + && request.pass_through != Some(PassThrough::Listed) + { + let mut ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + ctx.excluded_ids = &inputs.excluded_ids; + ctx.skipped_ids = &inputs.skipped_ids; + ctx.target_order = inputs.target_order.or(request_order); + healthy = self.host.callback_filters(&ctx, healthy).await?; + } + if inputs.count_only { + return Ok(Candidates { + name: name.to_owned(), + model_group: resolution.group, + kind: resolution.kind, + all, + healthy, + wildcard_request, + }); + } + + if listed { + if settings.enable_pre_call_checks && !resolution.kind.is_single() { + healthy = self + .pre_call_checks( + snapshot, + op, + request, + &resolution.group, + name, + &all, + healthy, + ) + .await?; + } + // Only a priority group narrows a listed pass-through pick by order. + let priority = resolution.kind == ResolveKind::RoutingGroup + && registry.routing_groups().iter().any(|g| { + g.group_name == resolution.group && crate::routing_groups::is_priority(g) + }); + if priority { + let order = request + .target_order + .or_else(|| healthy.iter().filter_map(|d| d.order()).min()); + if let Some(order) = order { + healthy.retain(|d| d.order() == Some(order)); + } + } + return Ok(Candidates { + name: name.to_owned(), + model_group: resolution.group, + kind: resolution.kind, + all, + healthy, + wildcard_request, + }); + } + + if !settings.plugins.is_empty() && !healthy.is_empty() { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + healthy = self + .host + .select(&SelectStage::Plugins, &ctx, healthy) + .await?; + } + + if settings.enable_pre_call_checks && !resolution.kind.is_single() { + healthy = self + .pre_call_checks( + snapshot, + op, + request, + &resolution.group, + name, + &all, + healthy, + ) + .await?; + } + + if request.filter_tags && !healthy.is_empty() { + let mut ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + ctx.pool = &all; + healthy = self.host.select(&SelectStage::Tags, &ctx, healthy).await?; + } + + if !resolution.kind.is_single() { + healthy = crate::routing_plugins::narrow(healthy, plugin_models.as_deref())?; + } + + // `_without_declared_text_only_deployments`: an image request skips the deployments that + // declare no vision, unless that leaves nothing. + if request.has_image && !resolution.kind.is_single() && !healthy.is_empty() { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + let text_only = self + .host + .select(&SelectStage::TextOnly, &ctx, healthy.clone()) + .await?; + let vision: Vec> = healthy + .iter() + .filter(|d| !text_only.iter().any(|t| t.id == d.id)) + .cloned() + .collect(); + if !vision.is_empty() { + healthy = vision; + } + } + + // `_without_decisions_deployments_refusing_media`: a Decisions call carrying audio or + // videos skips the deployments that refuse them, unless that leaves nothing. + if request.decisions_media && !resolution.kind.is_single() && !healthy.is_empty() { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + let refusing = self + .host + .select(&SelectStage::MediaRefusing, &ctx, healthy.clone()) + .await?; + let taking: Vec> = healthy + .iter() + .filter(|d| !refusing.iter().any(|r| r.id == d.id)) + .cloned() + .collect(); + if !taking.is_empty() { + healthy = taking; + } + } + + if let Some(pinned) = request.pinned_deployment_id.as_deref() + && healthy.iter().any(|d| d.id == pinned) + { + healthy.retain(|d| d.id == pinned); + } + + let order = inputs + .target_order + .or(request_order) + .or_else(|| healthy.iter().filter_map(|d| d.order()).min()); + if let Some(order) = order { + healthy.retain(|d| d.order() == Some(order)); + } + + healthy.retain(|d| { + !request.excluded_ids.contains(&d.id) && !inputs.excluded_ids.contains(&d.id) + }); + + if !inputs.skipped_ids.is_empty() || !request.retry_skipped_ids.is_empty() { + let kept: Vec> = healthy + .iter() + .filter(|d| { + !inputs.skipped_ids.contains(&d.id) + && !request.retry_skipped_ids.contains(&d.id) + }) + .cloned() + .collect(); + if !kept.is_empty() { + healthy = kept; + } + } + + if request.pass_through == Some(PassThrough::Healthy) + && !resolution.kind.is_single() + && !healthy.is_empty() + { + healthy.retain(|d| uses_pass_through(d)); + if healthy.is_empty() { + return Err(Error::NoPassThroughDeployments { + model: name.to_owned(), + }); + } + } + + Ok(Candidates { + name: name.to_owned(), + model_group: resolution.group, + kind: resolution.kind, + all, + healthy, + wildcard_request, + }) + } + + /// `_pre_call_checks` over `healthy`, with the host's facts. + #[allow(clippy::too_many_arguments)] + async fn pre_call_checks( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &RouterRequest, + model_group: &str, + name: &str, + pool: &[Arc], + healthy: Vec>, + ) -> Result>, Error> { + let mut ctx = self.context( + snapshot, + op, + request, + model_group, + name, + StrategyScope::Default, + ); + ctx.pool = pool; + let facts = self.host.pre_call_facts(&ctx, &healthy).await?; + crate::precall::filter(model_group, healthy, pool, &facts) + } + + /// Pick one of `candidates.healthy` with the group's strategy. + pub(crate) async fn pick( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &RouterRequest, + candidates: &Candidates, + ) -> Result, Error> { + let no_deployment = || Error::NoHealthyDeployments { + model: candidates.failed_model(&snapshot.registry), + }; + // A deployment named by its id is returned as is, before any strategy runs. + if candidates.kind.is_single() { + return candidates + .healthy + .first() + .cloned() + .ok_or_else(no_deployment); + } + let (strategy, scope) = routing_context( + &snapshot.registry, + &snapshot.settings.routing_strategy, + request.routing_strategy.as_deref(), + &candidates.kind, + &candidates.model_group, + ); + if strategy == RoutingStrategy::Unset { + return Err(no_deployment()); + } + if strategy == RoutingStrategy::SimpleShuffle { + let weights = crate::weights::validate(request.router_weights.as_ref()) + .map_err(|_| Error::InvalidRouterWeights)?; + return simple_shuffle( + &candidates.healthy, + &candidates.model_group, + weights.as_ref(), + self.rng.as_ref(), + )? + .ok_or_else(no_deployment); + } + let ctx = self.context( + snapshot, + op, + request, + &candidates.model_group, + &candidates.name, + scope, + ); + self.host + .select( + &SelectStage::Strategy(strategy), + &ctx, + candidates.healthy.clone(), + ) + .await? + .into_iter() + .next() + .ok_or_else(no_deployment) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + struct Fixed(f64); + + impl Randomness for Fixed { + fn next_f64(&self) -> f64 { + self.0 + } + } + + fn deployment(id: &str, params: serde_json::Value) -> Arc { + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": params, "model_info": {"id": id}, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + #[rstest] + #[case::weight_low_draw(0.1, "a")] + #[case::weight_high_draw(0.9, "b")] + fn shuffle_uses_weight_before_rpm(#[case] draw: f64, #[case] expected: &str) { + let deployments = [ + deployment("a", json!({"model": "openai/a", "weight": 1, "rpm": 1000})), + deployment("b", json!({"model": "openai/b", "weight": 3})), + ]; + let picked = simple_shuffle(&deployments, "g", None, &Fixed(draw)) + .unwrap() + .unwrap(); + assert_eq!(picked.id, expected); + } + + #[rstest] + fn shuffle_prefers_request_weights() { + let deployments = [ + deployment("a", json!({"model": "openai/a", "weight": 100})), + deployment("b", json!({"model": "openai/b"})), + ]; + let weights = BTreeMap::from([("g".to_owned(), BTreeMap::from([("b".to_owned(), 1.0)]))]); + let picked = simple_shuffle(&deployments, "g", Some(&weights), &Fixed(0.99)) + .unwrap() + .unwrap(); + assert_eq!(picked.id, "b"); + } +} diff --git a/litellm-rust/crates/router-core/src/settings.rs b/litellm-rust/crates/router-core/src/settings.rs new file mode 100644 index 00000000000..f55b059d71c --- /dev/null +++ b/litellm-rust/crates/router-core/src/settings.rs @@ -0,0 +1,133 @@ +//! [`Settings`]: one typed, frozen view of the configuration a caller may read, built from the +//! static config and the current dynamic snapshot. + +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use crate::{ + config::{ + AllowedFailsPolicy, DEFAULT_COOLDOWN_TIME, PluginSpec, RetryPolicy, RouterConfig, + RouterDynamicConfig, RoutingStrategy, + }, + error::AttemptError, +}; + +#[derive(Clone, Debug)] +pub struct Settings { + pub routing_strategy: RoutingStrategy, + pub plugins: Arc<[PluginSpec]>, + pub cooldown_time: Duration, + pub disable_cooldowns: bool, + pub allowed_fails: Option, + pub allowed_fails_policy: Option, + pub retry_policy: Option, + pub model_group_retry_policy: Arc>, + pub num_retries: u32, + pub max_fallbacks: u32, + pub retry_after: Duration, + pub timeout: Option, + pub drop_params: Option, + pub enable_pre_call_checks: bool, + pub routing_plugins: bool, + pub callback_filters: bool, + pub enable_health_check_routing: bool, + pub health_check_staleness: Duration, + pub background_health_check_model_groups: Option>, + pub enable_weighted_failover: bool, + pub fallback_on_unsupported_params: bool, + pub provider_default_deployment_ids: Arc<[String]>, + pub dynamic: Arc, +} + +/// `DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER`. +pub const DEFAULT_HEALTH_CHECK_STALENESS: Duration = Duration::from_secs(300 * 2); + +fn seconds(value: f64) -> Duration { + Duration::try_from_secs_f64(value).unwrap_or_default() +} + +impl Settings { + pub fn new(config: &RouterConfig, dynamic: Arc) -> Self { + Self { + routing_strategy: config.routing_strategy.clone(), + plugins: config.plugins.clone().into(), + cooldown_time: config.cooldown_time.map_or(DEFAULT_COOLDOWN_TIME, seconds), + disable_cooldowns: config.disable_cooldowns, + allowed_fails: dynamic.allowed_fails, + allowed_fails_policy: config.allowed_fails_policy.clone(), + retry_policy: config.retry_policy.clone(), + model_group_retry_policy: Arc::new(config.model_group_retry_policy.clone()), + num_retries: dynamic.num_retries.unwrap_or(0), + max_fallbacks: dynamic.max_fallbacks, + retry_after: seconds(config.retry_after), + timeout: config.request_timeout.or(config.timeout).map(seconds), + drop_params: config.drop_params, + enable_pre_call_checks: config.enable_pre_call_checks, + routing_plugins: config.routing_plugins, + callback_filters: config.callback_filters, + enable_health_check_routing: config.enable_health_check_routing, + health_check_staleness: config + .health_check_staleness_threshold + .map_or(DEFAULT_HEALTH_CHECK_STALENESS, seconds), + background_health_check_model_groups: config + .background_health_check_model_groups + .clone() + .map(Into::into), + enable_weighted_failover: config.enable_weighted_failover, + fallback_on_unsupported_params: config.fallback_on_unsupported_params.unwrap_or(true), + provider_default_deployment_ids: config.provider_default_deployment_ids.clone().into(), + dynamic, + } + } + + /// `allowed_fails_for`: the policy's count for this failure. + pub fn allowed_fails_for(&self, failure: &AttemptError) -> Option { + self.allowed_fails_policy + .as_ref() + .and_then(|policy| policy.allowed_fails_for(failure)) + } + + /// How long an allowed-fails counter lives: `settings().cooldown_time or + /// DEFAULT_COOLDOWN_TIME_SECONDS`, so a router cooldown of 0 still counts over the default. + pub fn fails_window(&self) -> Duration { + if self.cooldown_time.is_zero() { + DEFAULT_COOLDOWN_TIME + } else { + self.cooldown_time + } + } + + /// `_is_allowed_fails_set_on_router`. + pub fn allowed_fails_set(&self) -> bool { + self.allowed_fails.is_some() && !self.dynamic.allowed_fails_is_default + } + + /// Python's `allowed_fails` default when set without a value (`litellm.allowed_fails`). + pub fn effective_allowed_fails(&self) -> u32 { + self.allowed_fails.unwrap_or(3) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::passed_to_the_router(Some(3), false, true)] + #[case::the_process_default(Some(3), true, false)] + #[case::none(None, false, false)] + fn allowed_fails_counts_as_set_unless_it_is_the_process_default( + #[case] allowed_fails: Option, + #[case] allowed_fails_is_default: bool, + #[case] expected: bool, + ) { + let dynamic = RouterDynamicConfig { + allowed_fails, + allowed_fails_is_default, + ..RouterDynamicConfig::default() + }; + let settings = Settings::new(&RouterConfig::default(), Arc::new(dynamic)); + assert_eq!(settings.allowed_fails_set(), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/strategy.rs b/litellm-rust/crates/router-core/src/strategy.rs new file mode 100644 index 00000000000..0e1f3b49fa8 --- /dev/null +++ b/litellm-rust/crates/router-core/src/strategy.rs @@ -0,0 +1,110 @@ +//! Which routing strategy picks for a request, and whose state it reads: a port of +//! `Router._get_routing_context`. A per-request `routing_strategy` override wins, then the +//! routing group the resolved model belongs to (called as the group, or as a member of a +//! non-priority group), then the router's own strategy. + +use crate::{ + config::RoutingStrategy, + registry::{Registry, ResolveKind}, +}; + +/// The per-request overrides `_OVERRIDABLE_ROUTING_STRATEGIES` honours: simple-shuffle and every +/// strategy with a default selector. Anything else is ignored. +pub const OVERRIDABLE: [&str; 6] = [ + "simple-shuffle", + "least-busy", + "usage-based-routing", + "usage-based-routing-v2", + "latency-based-routing", + "cost-based-routing", +]; + +/// Whose selector state a strategy pick reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StrategyScope { + /// The router's own strategy (the implicit `"default"` group). + Default, + /// A routing group's own selector. + Group(String), + /// A per-request override: the default selector when it names the router's strategy, else + /// one selector per strategy kept for the router's life. + Override, +} + +impl StrategyScope { + /// The scope as the host step spells it. + pub fn wire(&self) -> (&'static str, Option<&str>) { + match self { + Self::Default => ("default", None), + Self::Group(name) => ("group", Some(name)), + Self::Override => ("override", None), + } + } +} + +/// A request's `routing_strategy` override, when it is one Python honours. +pub fn request_override(raw: Option<&str>) -> Option { + let raw = raw?; + OVERRIDABLE + .contains(&raw) + .then(|| raw.parse().unwrap_or_default()) +} + +/// The strategy and selector scope for a pick of `model_group` (the resolved group, after +/// aliases), reached through a resolution of `kind`. +pub fn routing_context( + registry: &Registry, + default: &RoutingStrategy, + override_strategy: Option<&str>, + kind: &ResolveKind, + model_group: &str, +) -> (RoutingStrategy, StrategyScope) { + if let Some(strategy) = request_override(override_strategy) { + return (strategy, StrategyScope::Override); + } + let group = match kind { + ResolveKind::RoutingGroup => registry + .routing_groups() + .iter() + .find(|group| group.group_name == model_group), + ResolveKind::Alias | ResolveKind::Group => registry.member_routing_group(model_group), + ResolveKind::SpecificDeployment + | ResolveKind::Id + | ResolveKind::Default + | ResolveKind::Wildcard => None, + }; + match group { + None => (default.clone(), StrategyScope::Default), + Some(group) if is_priority(&group.routing_strategy) => { + (RoutingStrategy::SimpleShuffle, StrategyScope::Default) + } + Some(group) => ( + group.routing_strategy.clone(), + StrategyScope::Group(group.group_name.clone()), + ), + } +} + +fn is_priority(strategy: &RoutingStrategy) -> bool { + matches!(strategy, RoutingStrategy::Other(name) if name == "priority") +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::honoured(Some("least-busy"), Some(RoutingStrategy::LeastBusy))] + #[case::shuffle(Some("simple-shuffle"), Some(RoutingStrategy::SimpleShuffle))] + #[case::lar1_ignored(Some("lar1"), None)] + #[case::unknown_ignored(Some("bogus"), None)] + #[case::unset(None, None)] + fn only_overridable_strategies_override( + #[case] raw: Option<&str>, + #[case] expected: Option, + ) { + assert_eq!(request_override(raw), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/weights.rs b/litellm-rust/crates/router-core/src/weights.rs new file mode 100644 index 00000000000..3e5d76844ab --- /dev/null +++ b/litellm-rust/crates/router-core/src/weights.rs @@ -0,0 +1,81 @@ +//! `_router_weights` as `validate_router_weights` reads it (`litellm/types/router_weights.py`): +//! model group to deployment id to a finite weight of at least 0 (an int or a float, never a +//! bool or a string), with non-blank ids and at least one positive weight in every non-empty group. + +use std::collections::BTreeMap; + +use serde_json::Value; + +/// Validated request weights. +pub type RouterWeights = BTreeMap>; + +/// The request's weights didn't validate; the host raises the validator's own error. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InvalidRouterWeights; + +fn identifier(text: &str) -> bool { + text.chars().any(|c| !c.is_whitespace()) +} + +fn weight(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_f64().filter(|w| w.is_finite() && *w >= 0.0), + _ => None, + } +} + +/// `validate_router_weights(raw)`: `None` when the request passed none. +pub fn validate(raw: Option<&Value>) -> Result, InvalidRouterWeights> { + let groups = match raw { + None | Some(Value::Null) => return Ok(None), + Some(Value::Object(groups)) => groups, + Some(_) => return Err(InvalidRouterWeights), + }; + let mut validated = RouterWeights::new(); + for (group, weights) in groups { + let Value::Object(weights) = weights else { + return Err(InvalidRouterWeights); + }; + if !identifier(group) { + return Err(InvalidRouterWeights); + } + let mut entries = BTreeMap::new(); + for (id, value) in weights { + if !identifier(id) { + return Err(InvalidRouterWeights); + } + entries.insert(id.clone(), weight(value).ok_or(InvalidRouterWeights)?); + } + if !entries.is_empty() && !entries.values().any(|w| *w > 0.0) { + return Err(InvalidRouterWeights); + } + validated.insert(group.clone(), entries); + } + Ok(Some(validated)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::all_zero(json!({"g": {"a": 0.0, "b": 0.0}}))] + #[case::negative(json!({"g": {"a": -1.0}}))] + #[case::text(json!({"g": {"a": "1"}}))] + #[case::boolean(json!({"g": {"a": true}}))] + #[case::blank_id(json!({"g": {" ": 1.0}}))] + #[case::not_a_mapping(json!(["g"]))] + fn invalid_weights_are_refused(#[case] raw: Value) { + assert_eq!(validate(Some(&raw)), Err(InvalidRouterWeights)); + } + + #[rstest] + #[case::empty_group(json!({"g": {}}))] + #[case::int_weight(json!({"g": {"a": 1, "b": 0}}))] + fn valid_weights_are_kept(#[case] raw: Value) { + assert!(validate(Some(&raw)).unwrap().is_some()); + } +} diff --git a/litellm-rust/crates/router-core/tests/call_loop.rs b/litellm-rust/crates/router-core/tests/call_loop.rs new file mode 100644 index 00000000000..9f41371f255 --- /dev/null +++ b/litellm-rust/crates/router-core/tests/call_loop.rs @@ -0,0 +1,1757 @@ +//! A call through the router end to end: selection, retries, fallbacks and cooldowns against a +//! scripted host. + +use std::{collections::VecDeque, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use litellm_router_core::{ + Attempt, AttemptError, Deployment, Error, FailureKind, FailureReport, Host, + InMemoryRouterCache, Op, Output, Payload, Router, RouterCache, RouterConfig, + RouterDynamicConfig, RouterEvent, RouterRequest, RoutingContext, SeededRandomness, SelectStage, + SuccessReport, + call::StreamOrigin, + fallback::Fallbacks, + health::{UsageMetric, current_minute, usage_key}, + host::UncountedRetries, + select::PassThrough, +}; +use parking_lot::Mutex; +use rstest::{fixture, rstest}; +use serde_json::json; + +#[derive(Default)] +struct ScriptedHost { + /// Failures handed out in order, keyed by model group; a group with none left succeeds. + script: Mutex)>>, + events: Mutex>, + cache: InMemoryRouterCache, + /// Answer every success as a stream whose outcome the host reports when it ends. + deferred: bool, + /// [`Output::tracks_usage`] on every success. + tracks_usage: Option, + /// Refuse every fallback chain with a dict target. + refuses_dict_targets: bool, + /// Every fallback check raises this exception class, as a host's sync check does. + fallback_check_raises: Option<&'static str>, + /// Failures the tag stage raises in turn, before it answers. + pick_failures: Mutex>, + /// The next attempt's failure, whatever group it serves. + first_failure: Mutex>, +} + +impl ScriptedHost { + fn failing(group: &str, failures: Vec) -> Self { + Self::scripted(vec![(group, failures)]) + } + + fn scripted(groups: Vec<(&str, Vec)>) -> Self { + Self { + script: Mutex::new( + groups + .into_iter() + .map(|(group, failures)| (group.to_owned(), failures.into())) + .collect(), + ), + ..Self::default() + } + } + + fn selected(&self) -> Vec { + self.events + .lock() + .iter() + .filter_map(|event| match event { + RouterEvent::Selected { deployment_id, .. } => Some(deployment_id.clone()), + _ => None, + }) + .collect() + } + + fn retries(&self) -> usize { + self.events + .lock() + .iter() + .filter(|event| matches!(event, RouterEvent::Retry { .. })) + .count() + } +} + +#[async_trait] +impl Host for ScriptedHost { + async fn invoke(&self, attempt: &Attempt) -> Result { + if let Some(failure) = self.first_failure.lock().take() { + return Err(failure); + } + let next = self + .script + .lock() + .iter_mut() + .find(|(group, _)| *group == attempt.model_group) + .and_then(|(_, failures)| failures.pop_front()); + match next { + Some(failure) => Err(failure), + None => Ok(Output { + value: Payload(Arc::new(attempt.deployment.id.clone())), + total_tokens: 10, + blocked: None, + deferred: self.deferred, + tracks_usage: self.tracks_usage, + }), + } + } + + async fn select( + &self, + _stage: &SelectStage, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + match self.pick_failures.lock().pop_front() { + Some(failure) => Err(failure.into()), + None => Ok(candidates), + } + } + + async fn allow_fallback( + &self, + _request: &RouterRequest, + _target: &str, + ) -> Result { + match self.fallback_check_raises { + Some(class) => Err(AttemptError::host_raised(&[class])), + None => Ok(true), + } + } + + async fn refuses_fallbacks( + &self, + _level: u32, + targets: &[serde_json::Map], + ) -> bool { + self.refuses_dict_targets && !targets.is_empty() + } + + async fn sleep(&self, _duration: Duration) {} + + fn observe(&self, event: &RouterEvent) { + self.events.lock().push(event.clone()); + } + + fn cache(&self) -> Arc { + Arc::new(self.cache.clone()) + } +} + +#[fixture] +fn config() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/c"}, "model_info": {"id": "b1"}}, + ] + })) + .unwrap() +} + +fn router(config: RouterConfig, dynamic: serde_json::Value, host: Arc) -> Router { + let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap(); + Router::builder(config, dynamic, host) + .randomness(Arc::new(SeededRandomness::new(7))) + .build() + .unwrap() +} + +fn request() -> RouterRequest { + RouterRequest { + model: "primary".to_owned(), + ..RouterRequest::default() + } +} + +/// `primary` at order 1, its second deployment at order 2 when `tiered`, and two other groups. +fn tiers(tiered: bool) -> RouterConfig { + let second = if tiered { "primary" } else { "unused" }; + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/p1", "order": 1}, "model_info": {"id": "p1"}}, + {"model_name": second, "litellm_params": {"model": "openai/p2", "order": 2}, "model_info": {"id": "p2"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/b1"}, "model_info": {"id": "b1"}}, + {"model_name": "context", "litellm_params": {"model": "anthropic/c1"}, "model_info": {"id": "c1"}}, + ] + })) + .unwrap() +} + +fn failure(kind: FailureKind, status: u16) -> AttemptError { + AttemptError::new(kind, Some(status)) +} + +fn context_window() -> AttemptError { + failure(FailureKind::ContextWindowExceeded, 400) +} + +fn server_error() -> AttemptError { + failure(FailureKind::InternalServer, 500) +} + +fn agentic_followup() -> AttemptError { + let mut failed = server_error(); + failed.agentic_followup = true; + failed +} + +#[rstest] +#[tokio::test] +async fn a_rate_limit_retries_in_the_same_group(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::RateLimit, Some(429))], + )); + let router = router(config, json!({"num_retries": 2}), host.clone()); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!( + ( + response.model_group.as_str(), + response.attempted_retries, + response.attempted_fallbacks + ), + ("primary", 1, 0) + ); +} + +#[rstest] +#[tokio::test] +async fn a_bad_request_falls_back_without_retrying(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::BadRequest, Some(400))], + )); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let retried = host + .events + .lock() + .iter() + .any(|event| matches!(event, RouterEvent::Retry { .. })); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks, + retried + ), + ("b1", 1, false) + ); +} + +#[rstest] +#[tokio::test] +async fn a_rate_limit_cools_the_deployment_down(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::RateLimit, Some(429))], + )); + let router = router(config, json!({}), host.clone()); + let error = router + .call(Op::from("acompletion"), request()) + .await + .unwrap_err(); + let cooled = router + .health() + .active_cooldowns(&["a1".to_owned(), "a2".to_owned()]) + .await + .unwrap(); + assert_eq!( + (error.attempt().map(|failure| failure.kind), cooled.len()), + (Some(FailureKind::RateLimit), 1) + ); +} + +#[rstest] +#[case::now(false, (Some(1.0), (10, 1)))] +#[case::deferred(true, (None, (10, 1)))] +#[tokio::test] +async fn a_deferred_attempt_counts_its_request_and_tokens_but_no_success( + #[case] deferred: bool, + #[case] expected: (Option, (u64, u64)), +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a", "rpm": 100}, "model_info": {"id": "a1"}}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost { + deferred, + ..ScriptedHost::default() + }); + let router = router(config, json!({}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let deployment = router.registry().deployment("a1").unwrap(); + let successes = host + .cache + .get("a1:successes") + .and_then(|count| count.as_f64()); + let usage = router.health().usage(&deployment).await.unwrap(); + assert_eq!((successes, usage), expected); +} + +/// `litellm_params`, `model_info` and a limit only the host knows of all count; a realtime +/// session (`Some(false)` on a deferred answer) counts nothing, whatever limits it has. +#[rstest] +#[case::in_params(json!({"rpm": 100}), json!({}), false, None, (10, 1))] +#[case::in_model_info(json!({}), json!({"rpm": 100}), false, None, (10, 1))] +#[case::in_the_catalog(json!({}), json!({}), false, Some(true), (10, 1))] +#[case::no_limit(json!({}), json!({}), false, None, (0, 0))] +#[case::a_stream_in_model_info(json!({}), json!({"rpm": 100}), true, None, (10, 1))] +#[case::a_realtime_session(json!({"rpm": 100}), json!({}), true, Some(false), (0, 0))] +#[tokio::test] +async fn usage_counts_for_a_deployment_that_tracks_it( + #[case] params: serde_json::Value, + #[case] info: serde_json::Value, + #[case] deferred: bool, + #[case] tracks_usage: Option, + #[case] expected: (u64, u64), +) { + let mut litellm_params = json!({"model": "openai/a"}); + litellm_params + .as_object_mut() + .unwrap() + .extend(params.as_object().unwrap().clone()); + let mut model_info = json!({"id": "a1"}); + model_info + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": litellm_params, "model_info": model_info}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost { + deferred, + tracks_usage, + ..ScriptedHost::default() + }); + let router = router(config, json!({}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let deployment = router.registry().deployment("a1").unwrap(); + assert_eq!(router.health().usage(&deployment).await.unwrap(), expected); +} + +/// A stream end's report of a wildcard deployment counts under the model the route filled, the +/// key its request was counted under, not the configured `*`. +#[rstest] +#[tokio::test] +async fn a_reported_success_keeps_the_filled_wildcard_model() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "rpm": 100}, "model_info": {"id": "w1"}}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost::default()); + let router = router(config, json!({}), host.clone()); + let report = SuccessReport { + usage_model: Some("openai/gpt-4o".to_owned()), + ..SuccessReport::default() + }; + router.report_success("w1", 7.0, 0.0, report).await.unwrap(); + let minute = current_minute(std::time::SystemTime::now()); + let key = |model: &str| usage_key("w1", model, UsageMetric::Tpm, &minute); + assert_eq!( + ( + host.cache.get(&key("openai/gpt-4o")), + host.cache.get(&key("openai/*")), + ), + (Some(json!(7)), None), + ); +} + +/// `async_deployment_callback_on_failure`'s request is written only when it runs: a failure the +/// host says ran no failure callback (`recorded`) writes none, nor does a sync call's. +#[rstest] +#[case::the_first_logged_failure(false, false, 1, Some(1.0))] +#[case::a_later_failure_on_the_shared_logging_object(true, false, 0, None)] +#[case::a_sync_call(false, true, 0, Some(1.0))] +#[tokio::test] +async fn a_failed_attempt_counts_its_request_only_when_its_callback_ran( + #[case] recorded: bool, + #[case] uncounted_request: bool, + #[case] requests: u64, + #[case] fails: Option, +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + ] + })) + .unwrap(); + let mut failure = AttemptError::new(FailureKind::InternalServer, Some(500)); + failure.recorded = recorded; + failure.uncounted_request = uncounted_request; + let host = Arc::new(ScriptedHost::failing("primary", vec![failure])); + let router = router(config, json!({"num_retries": 0}), host.clone()); + let _ = router.call(Op::from("acompletion"), request()).await; + let deployment = router.registry().deployment("a1").unwrap(); + let usage = router.health().usage(&deployment).await.unwrap(); + let counted = host.cache.get("a1:fails").and_then(|count| count.as_f64()); + assert_eq!((usage.1, counted), (requests, fails)); +} + +#[rstest] +#[case::counted(true, 2)] +#[case::uncounted(false, 1)] +#[tokio::test] +async fn a_reported_failure_counts_its_request_when_asked( + #[case] counts_request: bool, + #[case] requests: u64, +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a", "rpm": 100}, "model_info": {"id": "a1"}}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost { + deferred: true, + ..ScriptedHost::default() + }); + let router = router(config, json!({}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let report = FailureReport { + counts_request, + ..FailureReport::default() + }; + router + .report_failure( + "a1", + &AttemptError::new(FailureKind::RateLimit, Some(429)), + report, + ) + .await + .unwrap(); + let deployment = router.registry().deployment("a1").unwrap(); + let fails = host.cache.get("a1:fails").and_then(|count| count.as_f64()); + let usage = router.health().usage(&deployment).await.unwrap(); + assert_eq!((fails, usage.1), (Some(1.0), requests)); +} + +#[rstest] +#[tokio::test] +async fn a_raising_retry_policy_lookup_ends_the_hop_and_falls_back(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::RateLimit, Some(429))], + )); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let request = RouterRequest { + retry_policy_raises: vec!["primary".to_owned()], + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + let events = host.events.lock(); + assert!( + !events + .iter() + .any(|e| matches!(e, RouterEvent::Retry { .. })) + ); + assert!(events.iter().any(|e| matches!( + e, + RouterEvent::RetryPolicyRaised { level: 0, model, .. } if model == "primary" + ))); + assert_eq!(response.model_group, "backup"); +} + +#[rstest] +#[tokio::test] +async fn a_num_retries_that_raises_ends_the_hop_and_falls_back(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let request = RouterRequest { + uncounted_retries: Some(UncountedRetries::Raises), + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + let events = host.events.lock(); + assert!( + !events + .iter() + .any(|e| matches!(e, RouterEvent::Retry { .. })) + ); + assert!(events.iter().any(|e| matches!( + e, + RouterEvent::RetryCountRaised { level: 0, model, .. } if model == "primary" + ))); + assert_eq!(response.model_group, "backup"); +} + +#[rstest] +#[case::negative(UncountedRetries::Negative)] +#[case::not_positive(UncountedRetries::NotPositive)] +#[tokio::test] +async fn a_num_retries_below_one_leaves_the_failure_standing( + config: RouterConfig, + #[case] uncounted: UncountedRetries, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(config, json!({"num_retries": 3}), host.clone()); + let request = RouterRequest { + uncounted_retries: Some(uncounted), + ..request() + }; + let refused = router.call(Op::from("acompletion"), request).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.status == Some(500))); + assert_eq!(host.retries(), 0); + assert!( + !host + .events + .lock() + .iter() + .any(|event| matches!(event, RouterEvent::RetryCountRaised { .. })) + ); +} + +#[rstest] +#[tokio::test] +async fn a_retry_policy_that_grants_retries_outranks_a_num_retries_that_raises( + config: RouterConfig, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + RouterConfig { + retry_policy: serde_json::from_value(json!({"DefaultRetries": 1})).unwrap(), + ..config + }, + json!({}), + host.clone(), + ); + let request = RouterRequest { + uncounted_retries: Some(UncountedRetries::Raises), + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + assert_eq!(response.model_group, "primary"); + assert_eq!(host.retries(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_name_nothing_serves_routes_to_the_default_deployment(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let router = router( + config, + json!({"default_deployment": { + "model_name": "dflt", "litellm_params": {"model": "openai/x"}, "model_info": {"id": "dd"} + }}), + host.clone(), + ); + let request = RouterRequest { + model: "other".to_owned(), + ..RouterRequest::default() + }; + let response = router + .call(Op::from("acompletion"), request.clone()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.model_group.as_str() + ), + ("dd", "other") + ); + let picked = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap(); + assert_eq!(picked.params.model, "other"); +} + +#[rstest] +#[tokio::test] +async fn a_stale_member_map_naming_a_dropped_group_raises_on_every_pick(mut config: RouterConfig) { + config.routing_group_members = Some([("primary".to_owned(), "gone".to_owned())].into()); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let refused = router.select(&Op::from("acompletion"), &request()).await; + assert!(matches!(refused, Err(Error::MissingRoutingGroup { group }) if group == "gone")); + let other = RouterRequest { + model: "backup".to_owned(), + ..RouterRequest::default() + }; + assert!( + router + .select(&Op::from("acompletion"), &other) + .await + .is_ok() + ); +} + +#[rstest] +#[tokio::test] +async fn written_indexes_replace_the_ones_the_deployments_build(config: RouterConfig) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let index: litellm_router_core::registry::IndexOverrides = serde_json::from_value(json!({ + "model_names": ["backup"], + "model_name": {"primary": [1], "backup": [2]}, + })) + .unwrap(); + router + .apply(litellm_router_core::ChangeSet { + index: Some(index), + ..litellm_router_core::ChangeSet::default() + }) + .unwrap(); + for _ in 0..4 { + let picked = router + .select(&Op::from("acompletion"), &request()) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } + router + .apply(litellm_router_core::ChangeSet { + index: Some(litellm_router_core::registry::IndexOverrides::default()), + ..litellm_router_core::ChangeSet::default() + }) + .unwrap(); + assert_eq!(router.registry().group("primary").unwrap().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_retry_policy_retries_a_pick_that_found_no_deployment(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let mut config = config; + config.retry_policy = serde_json::from_value(json!({"DefaultRetries": 2})).unwrap(); + let router = router(config, json!({"num_retries": 0}), host.clone()); + let request = RouterRequest { + model: "nothing".to_owned(), + ..RouterRequest::default() + }; + let refused = router.call(Op::from("acompletion"), request).await; + assert!(matches!(refused, Err(Error::UnknownModel { .. }))); + let events = host.events.lock(); + let retries = events + .iter() + .filter(|event| matches!(event, RouterEvent::Retry { .. })) + .count(); + assert_eq!(retries, 2); + assert!(events.iter().any(|event| matches!( + event, + RouterEvent::RetriesExhausted { + picked: true, + level: 0, + .. + } + ))); +} + +#[rstest] +#[tokio::test] +async fn a_default_deployment_python_cannot_copy_raises_for_a_name_nothing_serves( + config: RouterConfig, +) { + let router = router( + config, + json!({"default_deployment_raises": true}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + model: "other".to_owned(), + ..RouterRequest::default() + }; + let refused = router.select(&Op::from("acompletion"), &request).await; + assert!(matches!(refused, Err(Error::DefaultDeploymentRaises { model }) if model == "other")); + assert!( + router + .select(&Op::from("acompletion"), &self::request()) + .await + .is_ok() + ); +} + +#[rstest] +#[tokio::test] +async fn a_written_index_position_past_the_list_raises_for_that_name(config: RouterConfig) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let index: litellm_router_core::registry::IndexOverrides = + serde_json::from_value(json!({"model_name": {"primary": [0, 9], "backup": [2]}})).unwrap(); + router + .apply(litellm_router_core::ChangeSet { + index: Some(index), + ..litellm_router_core::ChangeSet::default() + }) + .unwrap(); + let refused = router.select(&Op::from("acompletion"), &request()).await; + assert!(matches!(refused, Err(Error::IndexOutOfRange { model }) if model == "primary")); + let backup = RouterRequest { + model: "backup".to_owned(), + ..RouterRequest::default() + }; + assert!( + router + .select(&Op::from("acompletion"), &backup) + .await + .is_ok() + ); +} + +#[rstest] +#[case::no_list_takes_the_order_tier(json!(null), true, false, "p2")] +#[case::unmatched_list_takes_the_order_tier(json!([{"unrelated": ["context"]}]), true, false, "p2")] +#[case::unmatched_list_falls_through_to_regular(json!([{"unrelated": ["context"]}]), false, false, "b1")] +#[case::unmatched_list_leaves_the_client_side_list(json!([{"unrelated": ["context"]}]), false, true, "b1")] +#[case::matched_list_beats_the_order_tier(json!([{"primary": ["context"]}]), true, false, "c1")] +#[case::matched_star_beats_the_order_tier(json!([{"*": ["context"]}]), true, false, "c1")] +#[case::matched_list_beats_the_client_side_list(json!([{"primary": ["context"]}]), false, true, "c1")] +#[case::matched_list_beats_both(json!([{"primary": ["context"]}]), true, true, "c1")] +#[tokio::test] +async fn a_context_window_error_takes_its_own_chain_only_when_the_group_matches( + #[case] context_window_fallbacks: serde_json::Value, + #[case] tiered: bool, + #[case] client_side: bool, + #[case] expected: &str, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![context_window()])); + let router = router( + tiers(tiered), + json!({ + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": context_window_fallbacks, + }), + host.clone(), + ); + let call = RouterRequest { + fallbacks: client_side.then(|| Some(serde_json::from_value(json!(["backup"])).unwrap())), + ..request() + }; + let response = router.call(Op::from("acompletion"), call).await.unwrap(); + assert_eq!(response.deployment_id, expected); +} + +#[rstest] +#[case::context_window_keeps_its_own_error( + context_window(), + FailureKind::ContextWindowExceeded, + false +)] +#[case::any_other_raises_the_last_hop(server_error(), FailureKind::ServiceUnavailable, true)] +#[tokio::test] +async fn a_failed_order_chain_raises_by_the_error_that_started_it( + #[case] first: AttemptError, + #[case] raised: FailureKind, + #[case] last_hop: bool, +) { + let host = Arc::new(ScriptedHost::scripted(vec![ + ("primary", vec![first, failure(FailureKind::RateLimit, 429)]), + ( + "backup", + vec![failure(FailureKind::ServiceUnavailable, 503)], + ), + ])); + let router = router( + tiers(true), + json!({ + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": [{"unrelated": ["context"]}], + }), + host.clone(), + ); + let refused = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.kind == raised)); + assert_eq!(host.selected(), ["p1", "p2", "b1"]); + assert!( + host.events + .lock() + .contains(&RouterEvent::OrderChainFailed { level: 0, last_hop }) + ); +} + +#[rstest] +#[case::context_window_skips_it(context_window(), "backup")] +#[case::a_rate_limit_takes_it(failure(FailureKind::RateLimit, 429), "primary")] +#[tokio::test] +async fn weighted_failover_serves_every_failure_but_a_context_window_error( + #[case] first: AttemptError, + #[case] served_by: &str, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![first])); + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/c"}, "model_info": {"id": "b1"}}, + ], + "enable_weighted_failover": true, + })) + .unwrap(); + let router = router( + config, + json!({ + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": [{"unrelated": ["backup"]}], + }), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.model_group, served_by); +} + +#[rstest] +#[tokio::test] +async fn a_failed_agentic_followup_is_not_retried_but_still_falls_back(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![agentic_followup()])); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.deployment_id, "b1"); + assert_eq!(host.retries(), 0); + assert_eq!(host.selected().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_failed_agentic_followup_skips_the_retry_policy_lookup(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![agentic_followup()])); + let router = router(config, json!({"num_retries": 3}), host.clone()); + let call = RouterRequest { + retry_policy_raises: vec!["primary".to_owned()], + ..request() + }; + let refused = router.call(Op::from("acompletion"), call).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert!( + !host + .events + .lock() + .iter() + .any(|event| matches!(event, RouterEvent::RetryPolicyRaised { .. })) + ); +} + +#[rstest] +#[tokio::test] +async fn a_retry_that_fails_its_agentic_followup_ends_the_retries(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![server_error(), agentic_followup()], + )); + let router = router(config, json!({"num_retries": 3}), host.clone()); + let refused = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert_eq!((host.retries(), host.selected().len()), (1, 2)); +} + +fn stream_origin() -> StreamOrigin { + StreamOrigin { + group: "primary".to_owned(), + ..StreamOrigin::default() + } +} + +fn stream_request() -> RouterRequest { + RouterRequest { + num_retries: Some(3), + ..request() + } +} + +#[rstest] +#[tokio::test] +async fn a_stream_that_failed_its_agentic_followup_is_not_retried(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let router = router(config, json!({}), host.clone()); + let refused = router + .retry_stream( + Op::from("acompletion"), + stream_request(), + stream_origin(), + agentic_followup(), + ) + .await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert_eq!((host.retries(), host.selected().len()), (0, 0)); +} + +#[rstest] +#[tokio::test] +async fn a_stream_retry_that_fails_its_agentic_followup_ends_the_retries(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![agentic_followup()])); + let router = router(config, json!({}), host.clone()); + let refused = router + .retry_stream( + Op::from("acompletion"), + stream_request(), + stream_origin(), + server_error(), + ) + .await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert_eq!((host.retries(), host.selected().len()), (1, 1)); +} + +#[rstest] +#[tokio::test] +async fn a_stream_retry_is_reported_before_its_attempt(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let router = router(config, json!({}), host.clone()); + let response = router + .retry_stream( + Op::from("acompletion"), + stream_request(), + stream_origin(), + server_error(), + ) + .await + .unwrap(); + assert_eq!((response.attempted_retries, host.retries()), (1, 1)); +} + +fn fallback_list(value: serde_json::Value) -> Fallbacks { + serde_json::from_value(value).unwrap() +} + +#[rstest] +#[case::names(json!(["backup"]))] +#[case::keyed(json!([{"primary": ["backup"]}]))] +#[case::star(json!([{"*": ["backup"]}]))] +fn a_request_fallback_list_can_take_over_a_stream( + config: RouterConfig, + #[case] list: serde_json::Value, +) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + fallbacks: Some(Some(fallback_list(list))), + ..request() + }; + let policy = router.stream_policy(&request, &stream_origin(), None); + assert!(policy.can_fall_back); + assert!(policy.refusal_fallback); +} + +#[rstest] +fn a_request_content_policy_list_can_take_over_a_stream(config: RouterConfig) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + content_policy_fallbacks: Some(Some(fallback_list(json!([{"primary": ["backup"]}])))), + ..request() + }; + let policy = router.stream_policy(&request, &stream_origin(), None); + assert!(policy.can_fall_back); + assert!(policy.refusal_fallback); +} + +#[rstest] +#[case::cleared(None)] +#[case::another_group(Some(json!([{"other": ["backup"]}])))] +fn a_request_fallback_list_replaces_the_router_list_for_a_stream( + config: RouterConfig, + #[case] list: Option, +) { + let router = router( + config, + json!({"fallbacks": [{"primary": ["backup"]}], "default_fallbacks": ["backup"]}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + fallbacks: Some(list.map(fallback_list)), + ..request() + }; + let policy = router.stream_policy(&request, &stream_origin(), None); + assert!(!policy.can_fall_back); +} + +#[rstest] +#[case::readable(vec![], 5, true)] +#[case::malformed(vec!["primary".to_owned()], 1, false)] +fn a_group_policy_that_does_not_read_governs_no_stream_retry( + mut config: RouterConfig, + #[case] retry_policy_raises: Vec, + #[case] budget: u32, + #[case] applies: bool, +) { + config.retry_policy = serde_json::from_value(json!({"InternalServerErrorRetries": 5})).unwrap(); + let router = router( + config, + json!({"num_retries": 1}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + retry_policy_raises, + ..request() + }; + let policy = router.stream_policy( + &request, + &stream_origin(), + Some(FailureKind::InternalServer), + ); + assert_eq!(policy.retry_budget, Some((budget, applies))); +} + +#[rstest] +#[tokio::test] +async fn a_level_reports_its_fallback_handling_before_any_hop(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![context_window()])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let events = host.events.lock(); + let falling_back = RouterEvent::FallingBack { + level: 0, + model: "primary".to_owned(), + }; + let reported = events.iter().position(|e| *e == falling_back).unwrap(); + let hopped = events + .iter() + .position(|e| matches!(e, RouterEvent::Fallback { .. })) + .unwrap(); + assert!(reported < hopped); +} + +#[rstest] +#[tokio::test] +async fn the_order_path_reports_no_fallback_handling() { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(tiers(true), json!({"num_retries": 0}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert!( + !host + .events + .lock() + .iter() + .any(|e| matches!(e, RouterEvent::FallingBack { .. })) + ); +} + +fn fallbacks_failed(host: &ScriptedHost) -> Vec<(u32, bool, bool)> { + host.events + .lock() + .iter() + .filter_map(|event| match event { + RouterEvent::FallbacksFailed { + level, + chain, + last_hop, + } => Some((*level, *chain, *last_hop)), + _ => None, + }) + .collect() +} + +#[rstest] +#[tokio::test] +async fn a_chain_that_fails_reports_every_level_innermost_first(config: RouterConfig) { + let host = Arc::new(ScriptedHost::scripted(vec![ + ("primary", vec![server_error()]), + ("backup", vec![server_error()]), + ])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap_err(); + assert_eq!( + fallbacks_failed(&host), + vec![(1, true, false), (0, true, true)] + ); +} + +#[rstest] +#[tokio::test] +async fn a_chain_past_max_fallbacks_reports_the_levels_own_failure(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 0, "max_fallbacks": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap_err(); + assert_eq!(fallbacks_failed(&host), vec![(0, true, false)]); +} + +#[rstest] +#[tokio::test] +async fn a_served_fallback_and_a_disabled_walk_report_no_failure(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let dynamic = json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}); + let served = router(config.clone(), dynamic.clone(), host.clone()); + served + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(fallbacks_failed(&host), vec![]); + + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let disabled = router(config, dynamic, host.clone()); + let request = RouterRequest { + disable_fallbacks: true, + ..request() + }; + disabled + .call(Op::from("acompletion"), request) + .await + .unwrap_err(); + assert_eq!(fallbacks_failed(&host), vec![]); +} + +#[tokio::test] +async fn the_order_path_reports_no_failed_fallbacks() { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(tiers(true), json!({"num_retries": 0}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(fallbacks_failed(&host), vec![]); +} + +fn refusing_host(group: &str) -> Arc { + Arc::new(ScriptedHost { + refuses_dict_targets: true, + ..ScriptedHost::failing(group, vec![server_error()]) + }) +} + +#[rstest] +#[case::dict_target_first(json!([{"model": "backup", "api_key": "x"}, "backup"]))] +#[case::dict_target_last(json!(["backup", {"model": "backup", "api_key": "x"}]))] +#[tokio::test] +async fn a_refused_chain_tries_none_of_its_targets( + config: RouterConfig, + #[case] chain: serde_json::Value, +) { + let host = refusing_host("primary"); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": chain}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a("InternalServerError"))); + assert_eq!(host.selected().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_chain_of_names_is_never_refused(config: RouterConfig) { + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + refusing_host("primary"), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.deployment_id, "b1"); +} + +#[rstest] +#[tokio::test] +async fn a_refused_chain_on_the_order_path_raises_the_refusal() { + let host = refusing_host("primary"); + let router = router( + tiers(true), + json!({"num_retries": 0, "fallbacks": [{"primary": [{"model": "backup", "api_key": "x"}]}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a("ValueError"))); + assert_eq!(host.selected(), ["p1"]); +} + +#[rstest] +#[tokio::test] +async fn a_refused_chain_fails_only_its_own_level(mut config: RouterConfig) { + let host = Arc::new(ScriptedHost { + refuses_dict_targets: true, + ..ScriptedHost::scripted(vec![ + ("primary", vec![server_error()]), + ("backup", vec![server_error()]), + ]) + }); + let dynamic = json!({"num_retries": 0, "fallbacks": [ + {"primary": ["backup", "third"]}, + {"backup": [{"model": "third", "api_key": "x"}]}, + ]}); + config.model_list.push( + serde_json::from_value(json!({ + "model_name": "third", "litellm_params": {"model": "openai/t"}, "model_info": {"id": "t1"}, + })) + .unwrap(), + ); + let response = router(config, dynamic, host.clone()) + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.deployment_id, "t1"); +} + +fn shared_public_name() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "backup", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "b1"}}, + {"model_name": "t1-pub", "litellm_params": {"model": "openai/m"}, + "model_info": {"id": "p1", "team_id": "T1", "team_public_model_name": "pub"}}, + {"model_name": "t2-pub", "litellm_params": {"model": "openai/m"}, + "model_info": {"id": "p2", "team_id": "T2", "team_public_model_name": "pub"}}, + ] + })) + .unwrap() +} + +fn proxy_admin_of_pub() -> RouterRequest { + RouterRequest { + model: "pub".to_owned(), + proxy_admin: true, + admin_model: Some("pub".to_owned()), + ..RouterRequest::default() + } +} + +#[rstest] +#[tokio::test] +async fn a_name_several_teams_share_falls_back() { + let router = router( + shared_public_name(), + json!({"fallbacks": [{"pub": ["backup"]}]}), + Arc::new(ScriptedHost::default()), + ); + let response = router + .call(Op::from("acompletion"), proxy_admin_of_pub()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks + ), + ("b1", 1) + ); +} + +#[rstest] +#[tokio::test] +async fn a_name_several_teams_share_raises_its_refusal_when_the_fallback_fails() { + let router = router( + shared_public_name(), + json!({"fallbacks": [{"pub": ["backup"]}]}), + Arc::new(ScriptedHost::failing("backup", vec![server_error()])), + ); + let raised = router + .call(Op::from("acompletion"), proxy_admin_of_pub()) + .await; + assert!(matches!(raised, Err(Error::MultipleTeams { model }) if model == "pub")); +} + +/// A group whose second order level belongs to another team. +fn team_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/p1", "order": 1}, + "model_info": {"id": "p1", "team_id": "t1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/p2", "order": 2}, + "model_info": {"id": "p2", "team_id": "t2"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[case::listed_for_the_team(Some("t1"), false)] +#[case::listed_for_no_team(None, true)] +#[tokio::test] +async fn a_failed_level_lists_its_order_levels_for_the_metadata_team( + #[case] listing_team_id: Option<&str>, + #[case] walks_the_order_chain: bool, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(team_tiers(), json!({"num_retries": 0}), host.clone()); + let request = RouterRequest { + team_id: Some("t1".to_owned()), + listing_team_id: listing_team_id.map(str::to_owned), + ..request() + }; + let raised = router.call(Op::from("acompletion"), request).await; + assert_eq!( + matches!(raised, Err(Error::NoHealthyDeployments { .. })), + walks_the_order_chain + ); + assert_eq!(host.selected(), ["p1"]); +} + +fn tagged_request() -> RouterRequest { + RouterRequest { + filter_tags: true, + ..request() + } +} + +#[rstest] +#[tokio::test] +async fn a_failed_pick_retries_like_a_failed_attempt(config: RouterConfig) { + let host = Arc::new(ScriptedHost { + pick_failures: Mutex::new(VecDeque::from([failure(FailureKind::RateLimit, 429)])), + ..ScriptedHost::default() + }); + let response = router(config, json!({"num_retries": 1}), host.clone()) + .call(Op::from("acompletion"), tagged_request()) + .await + .unwrap(); + assert_eq!( + (response.model_group.as_str(), response.attempted_retries), + ("primary", 1) + ); +} + +#[rstest] +#[tokio::test] +async fn a_failed_pick_falls_back_with_no_deployment_to_blame(config: RouterConfig) { + let host = Arc::new(ScriptedHost { + pick_failures: Mutex::new(VecDeque::from([failure(FailureKind::RateLimit, 429)])), + ..ScriptedHost::default() + }); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), tagged_request()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks + ), + ("b1", 1) + ); + assert_eq!(host.selected(), ["b1"]); +} + +#[rstest] +#[tokio::test] +async fn a_fallback_check_that_raises_ends_the_chain_with_the_levels_own_error( + config: RouterConfig, +) { + let host = Arc::new(ScriptedHost { + fallback_check_raises: Some("TypeError"), + ..ScriptedHost::failing("primary", vec![failure(FailureKind::RateLimit, 429)]) + }); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.kind == FailureKind::RateLimit)); + assert_eq!(host.selected().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_fallback_check_that_raises_on_the_order_path_raises_its_own_error() { + let host = Arc::new(ScriptedHost { + fallback_check_raises: Some("TypeError"), + ..ScriptedHost::scripted(vec![("primary", vec![server_error(), server_error()])]) + }); + let router = router( + tiers(true), + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a("TypeError"))); +} + +#[rstest] +#[case::an_int(json!(5))] +#[case::none(json!(null))] +#[tokio::test] +async fn a_dict_target_with_a_non_string_model_is_a_hop_that_attempts_nothing( + config: RouterConfig, + #[case] model: serde_json::Value, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": [{"model": model}, "backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks + ), + ("b1", 2) + ); + assert_eq!(host.selected().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_dict_target_without_a_model_serves_the_group_that_failed_again(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": [{"temperature": 0.5}]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!( + (response.model_group.as_str(), response.attempted_fallbacks), + ("primary", 1) + ); +} + +fn weighted_failover() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + ], + "enable_weighted_failover": true, + })) + .unwrap() +} + +#[rstest] +#[case::a_bug_propagates(AttemptError::host_raised(&["TypeError", "Exception"]), "TypeError")] +#[case::a_provider_error_defers(failure(FailureKind::RateLimit, 429), "InternalServerError")] +#[case::no_deployment_defers( + AttemptError::host_raised(&["RouterRateLimitError", "ValueError"]), + "InternalServerError" +)] +#[tokio::test] +async fn weighted_failover_swallows_only_api_and_rate_limit_errors( + #[case] hop_failure: AttemptError, + #[case] raised_class: &str, +) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![server_error(), hop_failure], + )); + let raised = router(weighted_failover(), json!({"num_retries": 0}), host.clone()) + .call(Op::from("acompletion"), request()) + .await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a(raised_class))); + assert_eq!(host.selected().len(), 2); +} + +fn wildcard_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "order": 1}, "model_info": {"id": "w1"}}, + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "order": 2}, "model_info": {"id": "w2"}}, + ] + })) + .unwrap() +} + +fn alias_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "gg", "litellm_params": {"model": "openai/a", "order": 1}, "model_info": {"id": "w1"}}, + {"model_name": "gg", "litellm_params": {"model": "openai/b", "order": 2}, "model_info": {"id": "w2"}}, + ] + })) + .unwrap() +} + +fn team_public_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "t1a", "litellm_params": {"model": "openai/a", "order": 1}, + "model_info": {"id": "w1", "team_id": "T1", "team_public_model_name": "tp"}}, + {"model_name": "t1b", "litellm_params": {"model": "openai/b", "order": 2}, + "model_info": {"id": "w2", "team_id": "T1", "team_public_model_name": "tp"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[case::a_wildcard(wildcard_tiers(), json!({"num_retries": 0}), "openai/gpt-4o", None)] +#[case::an_alias( + alias_tiers(), + json!({"num_retries": 0, "model_group_alias": {"al": "gg"}}), + "al", + None +)] +#[case::a_team_public_name(team_public_tiers(), json!({"num_retries": 0}), "tp", Some("T1"))] +#[tokio::test] +async fn order_levels_walk_through_any_name_the_model_list_lists( + #[case] config: RouterConfig, + #[case] dynamic: serde_json::Value, + #[case] model: &str, + #[case] team: Option<&str>, +) { + let host = Arc::new(ScriptedHost::default()); + *host.first_failure.lock() = Some(failure(FailureKind::RateLimit, 429)); + let request = RouterRequest { + model: model.to_owned(), + team_id: team.map(str::to_owned), + listing_team_id: team.map(str::to_owned), + ..RouterRequest::default() + }; + let response = router(config, dynamic, host.clone()) + .call(Op::from("acompletion"), request) + .await + .unwrap(); + assert_eq!(response.deployment_id, "w2"); +} + +#[rstest] +#[tokio::test] +async fn a_team_wildcard_public_name_routes_for_its_team() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "t1-w", "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "tw1", "team_id": "T1", "team_public_model_name": "tw-*"}}, + {"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "gp"}}, + ] + })) + .unwrap(); + let request = RouterRequest { + model: "tw-gpt-4o".to_owned(), + team_id: Some("T1".to_owned()), + ..RouterRequest::default() + }; + let response = router(config, json!({}), Arc::new(ScriptedHost::default())) + .call(Op::from("acompletion"), request) + .await + .unwrap(); + assert_eq!(response.deployment_id, "tw1"); +} + +#[rstest] +#[tokio::test] +async fn a_paused_deployment_named_by_id_falls_back() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, + "model_info": {"id": "a1", "blocked": true}}, + {"model_name": "backup", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "b1"}}, + ] + })) + .unwrap(); + let request = RouterRequest { + model: "a1".to_owned(), + ..RouterRequest::default() + }; + let response = router( + config, + json!({"fallbacks": [{"a1": ["backup"]}]}), + Arc::new(ScriptedHost::default()), + ) + .call(Op::from("acompletion"), request) + .await + .unwrap(); + assert_eq!(response.deployment_id, "b1"); +} + +/// `primary` with `a1`'s `model_info` as given, and an unremarkable `a2`. +fn with_first_info(info: serde_json::Value) -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": info}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[tokio::test] +async fn a_web_search_request_skips_the_deployments_that_declare_no_support() { + let config = with_first_info(json!({"id": "a1", "supports_web_search": false})); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + web_search: true, + ..request() + }; + for _ in 0..16 { + let picked = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } +} + +#[rstest] +#[tokio::test] +async fn an_endpoint_a_deployment_refuses_routes_to_the_others() { + let config = with_first_info(json!({ + "id": "a1", + "enforce_supported_endpoints": true, + "supported_endpoints": ["/v1/embeddings"], + })); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + endpoint: Some("/v1/chat/completions".to_owned()), + ..request() + }; + for _ in 0..16 { + let picked = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } +} + +#[rstest] +#[tokio::test] +async fn an_endpoint_every_deployment_refuses_fails_the_pick() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [{ + "model_name": "primary", + "litellm_params": {"model": "openai/a"}, + "model_info": { + "id": "a1", + "enforce_supported_endpoints": true, + "supported_endpoints": ["/v1/embeddings"], + }, + }] + })) + .unwrap(); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + endpoint: Some("/v1/chat/completions".to_owned()), + ..request() + }; + let refused = router + .call(Op::from("acompletion"), request) + .await + .unwrap_err(); + assert!(matches!( + refused, + Error::EndpointNotSupported { model, message } + if model == "primary" && message + == "/v1/chat/completions is not supported by model=primary. \ + Supported endpoints: /v1/embeddings" + )); +} + +#[rstest] +#[tokio::test] +async fn an_order_in_model_info_narrows_the_pick() { + let mut config = with_first_info(json!({"id": "a1", "order": 2})); + config.model_list[1] + .model_info + .insert("order".to_owned(), json!("1")); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + for _ in 0..16 { + let picked = router + .select(&Op::from("acompletion"), &request()) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } +} + +#[rstest] +#[tokio::test] +async fn a_fallback_hop_drops_the_order_the_request_named() { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![failure(FailureKind::InternalServer, 500)], + )); + let router = router( + tiers(false), + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host, + ); + let request = RouterRequest { + target_order: Some(1), + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + assert_eq!(response.deployment_id, "b1"); +} + +#[rstest] +#[tokio::test] +async fn a_listed_pass_through_pick_names_the_group_an_alias_resolves_to(config: RouterConfig) { + let router = router( + config, + json!({"model_alias_map": {"al": "primary"}}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + model: "al".to_owned(), + pass_through: Some(PassThrough::Listed), + ..RouterRequest::default() + }; + let refused = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap_err(); + assert!(matches!( + refused, + Error::NoPassThroughDeployments { model } if model == "primary" + )); +} diff --git a/litellm/accounting/__init__.py b/litellm/accounting/__init__.py index 5d59a35a987..e69de29bb2d 100644 --- a/litellm/accounting/__init__.py +++ b/litellm/accounting/__init__.py @@ -1,59 +0,0 @@ -"""Public accounting contracts. Importing these values does not enable accounting.""" - -from litellm.accounting.measurements import measurements_for -from litellm.accounting.protocols import CostAggregator -from litellm.accounting.types import ( - AdmissionDecision, - AggregateQuery, - AggregateRow, - CostEvent, - DeliveryState, - DeliveryTarget, - Dimension, - DimensionFilter, - DimensionName, - EventKind, - Granularity, - InboundUsageEvent, - Measurement, - Metric, - MetricName, - Multiplier, - NamedCost, - OutboundUsageEvent, - Outcome, - PricedQuantity, - PricingSnapshot, - Rate, - Unpriced, - validate_aggregate_rows, -) - -__all__ = [ - "AdmissionDecision", - "AggregateQuery", - "AggregateRow", - "CostAggregator", - "CostEvent", - "DeliveryState", - "DeliveryTarget", - "Dimension", - "DimensionFilter", - "DimensionName", - "EventKind", - "Granularity", - "InboundUsageEvent", - "Measurement", - "Metric", - "MetricName", - "Multiplier", - "NamedCost", - "OutboundUsageEvent", - "Outcome", - "PricedQuantity", - "PricingSnapshot", - "Rate", - "Unpriced", - "measurements_for", - "validate_aggregate_rows", -] diff --git a/litellm/accounting/application.py b/litellm/accounting/application.py new file mode 100644 index 00000000000..1e5f23e2997 --- /dev/null +++ b/litellm/accounting/application.py @@ -0,0 +1,71 @@ +from typing import TYPE_CHECKING, Final + +from litellm.accounting.delivery_projection import entity_transactions, project_deliveries +from litellm.accounting.projection import Contribution +from litellm.proxy.db.budget_window_spend_writer import commit_window_spend_updates +from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key +from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction +from litellm.proxy.db.model_usage_rollup import queue_model_usage_upsert +from litellm.repositories.accounting_delivery_repository import DeliveryProjection, LedgerTransaction + +if TYPE_CHECKING: + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.utils import PrismaClient, ProxyLogging + + +async def apply_deliveries( + transaction: LedgerTransaction, + deliveries: tuple[DeliveryProjection, ...], + *, + prisma_client: "PrismaClient", + writer: "DBSpendUpdateWriter", + proxy_logging_obj: "ProxyLogging", +) -> tuple[Contribution, ...]: + from litellm.proxy.guardrails.usage_tracking import UsageUnitIncrement, UsageUnitKey, upsert_usage_unit_row + + contributions: Final = project_deliveries(deliveries) + await writer.apply_accounting_entities( + prisma_client, proxy_logging_obj, transaction, entity_transactions(contributions) + ) + for table in sorted(DAILY_SPEND_TABLES.values(), key=lambda item: item.name): + rows = tuple( + {**dict(item.key), **dict(item.values), **dict(item.increments)} + for item in contributions + if item.table == table.name + ) + merged = merge_by_conflict_key(table, rows) + for offset in range(0, len(merged), 100): + sql, params = build_bulk_upsert(table, merged[offset : offset + 100]) + async with db_span("upsert_daily_spend", table.name): + await transaction.execute_raw(sql, *params) + async with transaction.batch_() as batcher: + for item in contributions: + if item.table == "LiteLLM_DailyModelUsage": + queue_model_usage_upsert(batcher, dict(item.key), dict(item.increments)) + for item in contributions: + if item.table == "LiteLLM_DailyGuardrailUsageUnits": + key = dict(item.key) + counters = dict(item.increments) + await upsert_usage_unit_row( + prisma_client, + UsageUnitKey(key["guardrail_id"], key["date"], key["team_id"], key["api_key"], key["usage_unit"]), + UsageUnitIncrement(int(counters["units"]), float(counters["cost"]), int(counters["untracked_units"])), + outer_transaction=transaction, + ) + windows: Final = tuple(_window(item) for item in contributions if item.table == "LiteLLM_BudgetWindowSpend") + await commit_window_spend_updates(prisma_client, windows, outer_transaction=transaction) + return contributions + + +def _window(item: Contribution) -> WindowSpendTransaction: + key: Final = dict(item.key) + values: Final = dict(item.values) + return WindowSpendTransaction( + entity_type=key["entity_type"], + entity_id=key["entity_id"], + window_duration=key["window_duration"], + window_start=values["window_start"] or "", + spend=float(dict(item.increments)["spend"]), + started_at=values["started_at"], + ) diff --git a/litellm/accounting/delivery_projection.py b/litellm/accounting/delivery_projection.py new file mode 100644 index 00000000000..da63542d5b8 --- /dev/null +++ b/litellm/accounting/delivery_projection.py @@ -0,0 +1,78 @@ +from collections.abc import Iterable +from itertools import chain, groupby +from typing import TYPE_CHECKING, Final +from urllib.parse import quote + +from litellm.accounting.projection import Contribution, project +from litellm.repositories.accounting_delivery_repository import DeliveryProjection + + +def _destination(contribution: Contribution) -> tuple[str, tuple[tuple[str, str], ...], str]: + return ( + contribution.table, + contribution.key, + (dict(contribution.values).get("window_start") or "") + if contribution.table == "LiteLLM_BudgetWindowSpend" + else "", + ) + + +def _sum(contributions: tuple[Contribution, ...]) -> Contribution: + first: Final = contributions[0] + increments: Final = tuple( + (column, sum(dict(item.increments).get(column, 0) for item in contributions)) + for column in dict.fromkeys( + column for column, _ in chain.from_iterable(item.increments for item in contributions) + ) + ) + starts: Final = tuple( + value + for column, value in chain.from_iterable(item.values for item in contributions) + if column == "started_at" and value + ) + values: Final = tuple( + (column, min(starts) if column == "started_at" and starts else value) for column, value in first.values + ) + return Contribution(first.table, first.key, increments, values) + + +def project_deliveries(deliveries: Iterable[DeliveryProjection]) -> tuple[Contribution, ...]: + ordered: Final = sorted(chain.from_iterable(project(delivery.row) for delivery in deliveries), key=_destination) + return tuple(_sum(tuple(group)) for _, group in groupby(ordered, key=_destination)) + + +def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpdateTransactions": + return { + "user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_UserTable")), + "end_user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_EndUserTable")), + "key_list_transactions": dict(_entity_totals(contributions, "LiteLLM_VerificationToken")), + "team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamTable")), + "team_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamMembership")), + "org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationTable")), + "org_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationMembership")), + "project_list_transactions": dict(_entity_totals(contributions, "LiteLLM_ProjectTable")), + "tag_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TagTable")), + "agent_list_transactions": dict(_entity_totals(contributions, "LiteLLM_AgentsTable")), + "model_access_group_list_transactions": dict( + _entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable") + ), + } + + +def _entity_totals(contributions: tuple[Contribution, ...], table: str) -> tuple[tuple[str, float], ...]: + return tuple( + (_entity_key(item), float(dict(item.increments)["spend"])) for item in contributions if item.table == table + ) + + +def _entity_key(item: Contribution) -> str: + fields: Final = dict(item.key) + if item.table == "LiteLLM_TeamMembership": + return f"team_id::{fields['team_id']}::user_id::{fields['user_id']}" + if item.table == "LiteLLM_OrganizationMembership": + return f"org_id::{quote(fields['organization_id'], safe='')}::user_id::{quote(fields['user_id'], safe='')}" + return item.key[0][1] + + +if TYPE_CHECKING: + from litellm.proxy._types import DBSpendUpdateTransactions diff --git a/litellm/accounting/measurements.py b/litellm/accounting/measurements.py deleted file mode 100644 index 83011a280da..00000000000 --- a/litellm/accounting/measurements.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Final - -from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent - - -def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]: - """Return the complete contribution set for one event, ready for one apply call.""" - metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics - occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at - return tuple( - Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric) - for metric in metrics or () - ) diff --git a/litellm/accounting/metrics.py b/litellm/accounting/metrics.py new file mode 100644 index 00000000000..ade715d5d14 --- /dev/null +++ b/litellm/accounting/metrics.py @@ -0,0 +1,30 @@ +from typing import Final + +from prometheus_client import Gauge, Histogram + +from litellm.repositories.accounting_delivery_repository import DeliveryMetrics + +_PENDING: Final = Gauge("litellm_accounting_pending", "Pending SpendLogs deliveries", multiprocess_mode="livemax") +_PARKED: Final = Gauge("litellm_accounting_parked", "Parked SpendLogs deliveries", multiprocess_mode="livemax") +_OLDEST: Final = Gauge( + "litellm_accounting_oldest_pending_seconds", + "Age of the oldest pending SpendLogs delivery", + multiprocess_mode="livemax", +) +_BATCH_TIME: Final = Histogram("litellm_accounting_batch_seconds", "SpendLogs delivery worker batch time") +_BATCH_COUNT: Final = Gauge( + "litellm_accounting_batch_deliveries", "Retained batch deliveries", multiprocess_mode="livemax" +) +_TABLE_BYTES: Final = Gauge( + "litellm_accounting_delivery_table_bytes", "Delivery table size in bytes", multiprocess_mode="livemax" +) + + +def record_deliveries(metrics: DeliveryMetrics, seconds: float) -> None: + _PENDING.set(metrics.pending_count) + _PARKED.set(metrics.parked_count) + _OLDEST.set(max(0.0, metrics.oldest_pending_seconds)) + _BATCH_TIME.observe(seconds) + + _BATCH_COUNT.set(metrics.batch_count) + _TABLE_BYTES.set(metrics.table_bytes) diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py index fc3a89ae306..dd52b49f101 100644 --- a/litellm/accounting/projection.py +++ b/litellm/accounting/projection.py @@ -1,5 +1,5 @@ from collections.abc import Iterator, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timezone from itertools import chain from types import MappingProxyType @@ -24,8 +24,11 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN _FIELDS: Final = TypeAdapter(dict[str, object]) _STRINGS: Final = TypeAdapter(tuple[str, ...]) +_VALUES: Final = TypeAdapter(tuple[object, ...]) _ROWS: Final = TypeAdapter(tuple[dict[str, object], ...]) _TASK_TYPES: Final = frozenset(load_model_insight_tasks()) +_MAX_INDEXED_TEXT_BYTES: Final = 2000 +_MAX_COUNTER: Final = 2**40 @dataclass(frozen=True, slots=True) @@ -54,7 +57,8 @@ def _tags(row: Mapping[str, object]) -> tuple[str, ...]: value: Final = row.get("request_tags") if value is None: return () - return _STRINGS.validate_json(value) if isinstance(value, str) else _STRINGS.validate_python(value) + parsed: Final = _VALUES.validate_json(value) if isinstance(value, str) else _VALUES.validate_python(value) + return tuple(tag for tag in parsed if isinstance(tag, str)) def _date(row: Mapping[str, object]) -> str | None: @@ -191,13 +195,31 @@ def _window_contributions(row: Mapping[str, object], metadata: Mapping[str, obje def project(row: Mapping[str, object]) -> tuple[Contribution, ...]: metadata: Final = _fields(row.get("metadata")) - return ( + contributions: Final = ( *_entity_contributions(row, metadata), *_daily_contributions(row, metadata), *_window_contributions(row, metadata), *_model_contributions(row, metadata), *_guardrail_contributions(row, metadata), ) + return tuple(_capped(contribution) for contribution in contributions if _indexable(contribution)) + + +def _indexable(contribution: Contribution) -> bool: + texts: Final = (value for _, value in (*contribution.key, *contribution.values) if value is not None) + return all(len(text.encode("utf-8", "surrogatepass")) <= _MAX_INDEXED_TEXT_BYTES for text in texts) + + +def _capped(contribution: Contribution) -> Contribution: + """Counters are summed into bigint columns in the charge's transaction, so one capped far below that range + can never overflow a day's total.""" + return replace( + contribution, + increments=tuple( + (name, max(-_MAX_COUNTER, min(_MAX_COUNTER, value)) if isinstance(value, int) else value) + for name, value in contribution.increments + ), + ) def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: diff --git a/litellm/accounting/protocols.py b/litellm/accounting/protocols.py deleted file mode 100644 index f6ec61983a0..00000000000 --- a/litellm/accounting/protocols.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from typing import Protocol - -from litellm.accounting.types import AggregateQuery, AggregateRow - - -class CostAggregator(Protocol): - async def apply(self, rows: tuple[Mapping[str, object], ...]) -> None: - """Apply stored request contributions atomically with their delivery receipts.""" - ... - - async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]: - """Return disjoint [start, end) rows per group using existing daily projections. - - Filters combine with AND, values within a filter with IN. DAY emits daily - buckets; None emits whole-interval totals. Reject unsupported metric/scope - combinations. Different groups may overlap, including non-additive tags. - """ - ... diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py index 51c7bbad81b..d5652f79ed0 100644 --- a/litellm/accounting/snapshot.py +++ b/litellm/accounting/snapshot.py @@ -2,12 +2,13 @@ from collections.abc import Callable, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure from litellm.proxy._types import SpendLogsPayload from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens -from litellm.proxy.spend_tracking.savings import compute_savings_spend, marks_gateway_injection +from litellm.proxy.spend_tracking.savings import SavingsSpend, compute_savings_spend, marks_gateway_injection if TYPE_CHECKING: from litellm.router import Router @@ -66,7 +67,47 @@ def build_accounting_payload( router: "Callable[[], Router | None]", ) -> Mapping[str, object]: metadata: Final = _FIELDS.validate_json(payload["metadata"]) - savings: Final = compute_savings_spend( + savings: Final = _savings_or_zero(payload, metadata, router) + accounting_metadata: Final = { + **{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS}, + "accounting_entities": entities, + "accounting_savings": savings._asdict(), + "accounting_model_access_groups": model_access_groups, + "accounting_windows": windows, + } + return _FIELDS.validate_python( + safe_json_structure( + { + **{ + key: value.isoformat() if isinstance(value, datetime) else value + for key, value in payload.items() + if key in _ACCOUNTING_FIELDS + }, + "metadata": accounting_metadata, + }, + non_finite_as_null=True, + ) + ) + + +def _savings_or_zero( + payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]" +) -> SavingsSpend: + """Savings are analytics computed from caller-supplied counts, so one that cannot be computed is stored as zero + rather than losing the charge it rides with.""" + try: + return _savings(payload, metadata, router) + except (ArithmeticError, ValueError, TypeError) as error: + verbose_proxy_logger.warning( + "Recording zero savings for request_id=%s, savings could not be computed: %s", payload["request_id"], error + ) + return SavingsSpend(compression=0.0, prompt_caching=0.0) + + +def _savings( + payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]" +) -> SavingsSpend: + return compute_savings_spend( model=payload["model"], custom_llm_provider=payload["custom_llm_provider"], compression_saved_tokens=extract_compression_saved_tokens(metadata), @@ -80,18 +121,41 @@ def build_accounting_payload( recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")), billed_at=payload["endTime"], ) - accounting_metadata: Final = { - **{key: value for key, value in metadata.items() if key in _METADATA_FIELDS}, - "accounting_entities": entities, - "accounting_savings": savings._asdict(), - "accounting_model_access_groups": model_access_groups, - "accounting_windows": windows, + + +_GUARDRAIL_ENTRIES: Final = TypeAdapter(tuple[dict[str, object], ...]) +_GUARDRAIL_METADATA_FIELDS: Final = frozenset({"guardrail_information", "standard_logging_guardrail_information"}) +_GUARDRAIL_ACCOUNTING_FIELDS: Final = frozenset( + { + "guardrail_id", + "guardrail_name", + "guardrail_provider", + "guardrail_usage", + "guardrail_cost_by_unit", + "guardrail_cost_in_spend", } +) +_GUARDRAIL_RESPONSE: Final = TypeAdapter(dict[str, object]) + + +def _accounting_guardrail_entries(value: object) -> object: + try: + entries: Final = _GUARDRAIL_ENTRIES.validate_python((value,) if isinstance(value, dict) else value) + except ValidationError: + return None + return [_accounting_guardrail_entry(entry) for entry in entries] + + +def _accounting_guardrail_entry(entry: Mapping[str, object]) -> Mapping[str, object]: + kept: Final = {key: item for key, item in entry.items() if key in _GUARDRAIL_ACCOUNTING_FIELDS} + response: Final = entry.get("guardrail_response") + if not isinstance(response, dict) or "tokens_saved" not in response: + return kept return { - **{ - key: value.isoformat() if isinstance(value, datetime) else value - for key, value in payload.items() - if key in _ACCOUNTING_FIELDS - }, - "metadata": _FIELDS.validate_python(safe_json_structure(accounting_metadata, non_finite_as_null=True)), + **kept, + "guardrail_response": {"tokens_saved": _GUARDRAIL_RESPONSE.validate_python(response)["tokens_saved"]}, } + + +def _accounting_metadata_field(key: str, value: object) -> object: + return _accounting_guardrail_entries(value) if key in _GUARDRAIL_METADATA_FIELDS else value diff --git a/litellm/accounting/types.py b/litellm/accounting/types.py deleted file mode 100644 index 2ee6f60f756..00000000000 --- a/litellm/accounting/types.py +++ /dev/null @@ -1,428 +0,0 @@ -"""Accounting values use UTC instants, decimal strings on the wire and USD charges.""" - -from collections.abc import Mapping -from datetime import datetime, timezone -from decimal import Decimal -from enum import Enum -from typing import Annotated, Final, Literal, TypeAlias - -from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator -from typing_extensions import Self - -from litellm.types.utils import CallTypes - -_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True) -Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")] -Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)] - - -def _utc(value: datetime) -> datetime: - return value.astimezone(timezone.utc) - - -Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)] - - -class MetricName(str, Enum): - """Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens.""" - - REQUESTS = "requests" - SUCCESSFUL_REQUESTS = "successful_requests" - FAILED_REQUESTS = "failed_requests" - PROMPT_TOKENS = "prompt_tokens" - COMPLETION_TOKENS = "completion_tokens" - REASONING_TOKENS = "reasoning_tokens" - TOTAL_TOKENS = "total_tokens" - CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens" - CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens" - CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr" - SPEND = "spend" - - -class DimensionName(str, Enum): - KEY = "key" - USER = "user" - TEAM = "team" - ORGANIZATION = "organization" - END_USER = "end_user" - MODEL = "model" - MODEL_GROUP = "model_group" - DEPLOYMENT = "deployment" - PROVIDER = "provider" - TOOL = "tool" - AGENT = "agent" - PROJECT = "project" - MODEL_ACCESS_GROUP = "model_access_group" - API_BASE = "api_base" - SERVICE_TIER = "service_tier" - TAG = "tag" - - -class Outcome(str, Enum): - SUCCEEDED = "succeeded" - FAILED = "failed" - CANCELLED = "cancelled" - - -class Granularity(str, Enum): - DAY = "day" - - -class EventKind(str, Enum): - INBOUND_USAGE = "inbound_usage" - OUTBOUND_USAGE = "outbound_usage" - COST = "cost" - - -class DeliveryState(str, Enum): - PENDING = "pending" - APPLIED = "applied" - PARKED = "parked" - - -class DeliveryTarget(str, Enum): - """One member per existing table the projector writes; each is applied and receipted independently.""" - - KEY_SPEND = "LiteLLM_VerificationToken" - USER_SPEND = "LiteLLM_UserTable" - TEAM_SPEND = "LiteLLM_TeamTable" - TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership" - ORGANIZATION_SPEND = "LiteLLM_OrganizationTable" - ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership" - END_USER_SPEND = "LiteLLM_EndUserTable" - AGENT_SPEND = "LiteLLM_AgentsTable" - TAG_SPEND = "LiteLLM_TagTable" - PROJECT_SPEND = "LiteLLM_ProjectTable" - MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable" - BUDGET_SPEND = "LiteLLM_BudgetTable" - BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend" - DAILY_USER_SPEND = "LiteLLM_DailyUserSpend" - DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend" - DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend" - DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend" - DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend" - DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend" - DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage" - DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend" - AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend" - - -_SUBSET_PAIRS: Final = ( - (MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS), - (MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS), -) - - -def _check_subsets(values: Mapping[str, Decimal]) -> None: - for part, whole in _SUBSET_PAIRS: - if part.value in values and whole.value in values and values[part.value] > values[whole.value]: - raise ValueError(f"{part.value} cannot exceed {whole.value}") - - -class Metric(BaseModel): - model_config = _CONFIG - - name: MetricName - value: Quantity - unit: Literal["count", "USD"] = "count" - - @model_validator(mode="after") - def validate_unit(self) -> Self: - if (self.name is MetricName.SPEND) != (self.unit == "USD"): - raise ValueError("SPEND requires USD; usage metrics require count") - if self.unit == "count" and self.value != self.value.to_integral_value(): - raise ValueError("Count metrics require integral values") - return self - - -class Dimension(BaseModel): - model_config = _CONFIG - - name: DimensionName - value: Identifier - - -_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP}) - - -def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]: - if len(frozenset(values)) != len(values): - raise ValueError("Duplicate dimension values") - single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS) - if len(frozenset(single_names)) != len(single_names): - raise ValueError("Only tag and model access group dimensions may have multiple values") - return values - - -Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)] -_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS)) - - -def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]: - if len(frozenset(value.name for value in values)) != len(values): - raise ValueError("Duplicate metric names") - return values - - -Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)] - - -def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None: - if end < start or (end == start and not allow_empty): - raise ValueError("End must follow start") - - -class InboundUsageEvent(BaseModel): - model_config = _CONFIG - - event_id: Identifier - request_id: Identifier - call_type: CallTypes - started_at: Instant - ended_at: Instant - outcome: Outcome - dimensions: Dimensions - metrics: Metrics - - @model_validator(mode="after") - def validate_caller_activity(self) -> Self: - _interval(self.started_at, self.ended_at, allow_empty=True) - if any(metric.name not in _CALLER_METRICS for metric in self.metrics): - raise ValueError("Inbound events own only caller request metrics") - if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics): - raise ValueError("Inbound events count exactly one caller request") - for metric in self.metrics: - if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED): - raise ValueError("Successful request count must match outcome") - if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED): - raise ValueError("Failed request count must match outcome") - return self - - -class OutboundUsageEvent(BaseModel): - model_config = _CONFIG - - event_id: Identifier - attempt_id: Identifier - request_id: Identifier | None - retry_of_attempt_id: Identifier | None - call_type: CallTypes - started_at: Instant - ended_at: Instant - outcome: Outcome - dimensions: Dimensions - metrics: Metrics | None - - @model_validator(mode="after") - def validate_attempt_usage(self) -> Self: - _interval(self.started_at, self.ended_at, allow_empty=True) - if self.retry_of_attempt_id == self.attempt_id: - raise ValueError("An attempt cannot retry itself") - if self.metrics is not None and any( - metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics - ): - raise ValueError("Outbound events own only attempt usage metrics") - if self.metrics is not None: - _check_subsets({metric.name.value: metric.value for metric in self.metrics}) - return self - - -PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"] - - -class Rate(BaseModel): - model_config = _CONFIG - - measure: Identifier - unit: PricingUnit - usd_per_unit: Quantity - - -class PricedQuantity(BaseModel): - """A pricing input in its billing unit. Only seconds may be fractional.""" - - model_config = _CONFIG - - measure: Identifier - unit: PricingUnit - value: Quantity - - @model_validator(mode="after") - def validate_discrete_quantity(self) -> Self: - if self.unit != "second" and self.value != self.value.to_integral_value(): - raise ValueError("Discrete priced quantities require integral values") - return self - - -class Multiplier(BaseModel): - """A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5.""" - - model_config = _CONFIG - - name: Identifier - value: Identifier - factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)] - - -class NamedCost(BaseModel): - """A free-form extra charge by name, e.g. azure_model_router_flat_cost.""" - - model_config = _CONFIG - - name: Identifier - amount: Quantity - - -class PricingSnapshot(BaseModel): - """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change. - - Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload. - Retries reuse the committed charge and snapshot. - Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge. - Component sums and float tolerances belong to reconciliation, not this structural contract. - """ - - model_config = _CONFIG - - schema_version: Literal[1] = 1 - calculator_version: Identifier - model: Identifier - provider: Identifier | None - pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"] - priced_model_key: Identifier - multipliers: tuple[Multiplier, ...] = () - rates: tuple[Rate, ...] = () - usage: tuple[PricedQuantity, ...] = () - input_cost: Quantity | None = None - output_cost: Quantity | None = None - cache_read_cost: Quantity | None = None - cache_creation_cost: Quantity | None = None - reasoning_cost: Quantity | None = None - tool_usage_cost: Quantity | None = None - guardrail_cost: Quantity | None = None - additional_costs: tuple[NamedCost, ...] = () - original_cost: Quantity | None = None - discount_percent: Quantity | None = None - discount_amount: Quantity | None = None - margin_percent: Quantity | None = None - margin_fixed_amount: Quantity | None = None - margin_total_amount: Quantity | None = None - upstream_reported_cost: Quantity | None = None - total_cost: Quantity - - @model_validator(mode="after") - def validate_snapshot(self) -> Self: - rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates) - quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage) - if len(rate_keys) != len(self.rates): - raise ValueError("Duplicate rate measures and units") - if len(quantity_keys) != len(self.usage): - raise ValueError("Duplicate priced quantity measures and units") - if not rate_keys.issubset(quantity_keys): - raise ValueError("Every rate requires one priced quantity with the same measure and unit") - if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs): - raise ValueError("Duplicate additional cost names") - if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers): - raise ValueError("Duplicate multiplier names") - _check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"}) - return self - - -class CostEvent(BaseModel): - model_config = _CONFIG - - event_id: Identifier - usage_event_id: Identifier - occurred_at: Instant - dimensions: Dimensions - charges: Metrics - snapshot: PricingSnapshot - - @model_validator(mode="after") - def validate_charge(self) -> Self: - if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND: - raise ValueError("A cost event requires one authoritative USD spend metric") - if self.event_id == self.usage_event_id: - raise ValueError("Cost and usage events require distinct identities") - if self.snapshot.total_cost != self.charges[0].value: - raise ValueError("Snapshot total must equal the authoritative spend") - return self - - -class Unpriced(BaseModel): - model_config = _CONFIG - - usage_event_id: Identifier - reason: Identifier - - -class Measurement(BaseModel): - model_config = _CONFIG - - source_event_id: Identifier - occurred_at: Instant - dimensions: Dimensions - metric: Metric - - -class DimensionFilter(BaseModel): - model_config = _CONFIG - - name: DimensionName - values: Annotated[tuple[Identifier, ...], Field(min_length=1)] - - -class AggregateQuery(BaseModel): - model_config = _CONFIG - - metric: MetricName - filters: tuple[DimensionFilter, ...] - group_by: tuple[DimensionName, ...] - start: Instant - end: Instant - granularity: Granularity | None - - @model_validator(mode="after") - def validate_query(self) -> Self: - _interval(self.start, self.end) - if len(frozenset(item.name for item in self.filters)) != len(self.filters): - raise ValueError("Use one IN filter per dimension; filters combine with AND") - if len(frozenset(self.group_by)) != len(self.group_by): - raise ValueError("Duplicate group-by dimensions") - if any( - (instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0) - for instant in (self.start, self.end) - ): - raise ValueError("Initial reporting queries require UTC day boundaries") - return self - - -class AggregateRow(BaseModel): - model_config = _CONFIG - - start: Instant - end: Instant - dimensions: Dimensions - value: Quantity - - @model_validator(mode="after") - def validate_bounds(self) -> Self: - _interval(self.start, self.end) - return self - - -def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]: - """Validate [start, end) buckets per group; overlapping tag groups are not additive.""" - groups: Final = frozenset(frozenset(row.dimensions) for row in rows) - for group in groups: - ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start) - if any(left.end > right.start for left, right in zip(ordered, ordered[1:])): - raise ValueError("Aggregate buckets overlap within a dimension group") - return rows - - -class AdmissionDecision(BaseModel): - model_config = _CONFIG - - allowed: bool - reason: Identifier | None = None diff --git a/litellm/accounting/worker.py b/litellm/accounting/worker.py new file mode 100644 index 00000000000..df71f2da31c --- /dev/null +++ b/litellm/accounting/worker.py @@ -0,0 +1,102 @@ +import os +import time +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Protocol, + cast, # noqa: TID251 # generated Prisma transaction signatures cannot express the borrowed transaction protocol +) + +from litellm._logging import verbose_proxy_logger +from litellm.accounting.application import apply_deliveries +from litellm.accounting.metrics import record_deliveries +from litellm.accounting.projection import Contribution +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerDatabase, + LedgerTransaction, +) +from litellm.types.integrations.slack_alerting import AlertType + +if TYPE_CHECKING: + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.utils import PrismaClient, ProxyLogging + + +class _AccountingAlerts(Protocol): + async def alerting_handler( + self, message: str, level: Literal["Low", "Medium", "High"], alert_type: AlertType + ) -> None: ... + + +_JOB: Final = "accounting_delivery" + + +async def flush_accounting_deliveries( + writer: "DBSpendUpdateWriter", prisma_client: "PrismaClient", proxy_logging_obj: "ProxyLogging" +) -> None: + acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False) + if acquired is False: + return + applied: tuple[Contribution, ...] = () # rebind-ok: keep committed contributions for cache invalidation + + async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None: + nonlocal applied + applied = await apply_deliveries( # rebind-ok: invalidate caches only after the outer transaction commits + transaction, deliveries, prisma_client=prisma_client, writer=writer, proxy_logging_obj=proxy_logging_obj + ) + + try: + database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction + LedgerDatabase, prisma_client + ) + repository: Final = AccountingDeliveryRepository(database) + started: Final = time.monotonic() + batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "10000")) + + async def run_batch(*, retry_only: bool) -> bool: + result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only) + if result.applied: + await writer.invalidate_accounting_entities(applied, proxy_logging_obj) + if result.parked: + verbose_proxy_logger.error( + "Accounting delivery parked %d requests after isolated failures", len(result.parked) + ) + await _alert( + proxy_logging_obj, + message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.", + ) + return bool(result.applied or result.failed) + + await run_batch(retry_only=False) + retry_deadline: Final = time.monotonic() + 5 + for _ in range(batch_size): + if time.monotonic() >= retry_deadline or not await run_batch(retry_only=True): + break + await repository.cleanup_applied( + retention_days=int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_RETENTION_DAYS", "1")), + batch_size=batch_size * 2, + ) + metrics: Final = await repository.metrics() + record_deliveries(metrics, time.monotonic() - started) + if metrics.batch_count >= int( + os.environ.get("LITELLM_ACCOUNTING_BATCH_DELIVERY_ALERT_COUNT", "1000000") + ) or metrics.table_bytes >= int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_ALERT_BYTES", "1073741824")): + await _alert( + proxy_logging_obj, + message=f"Accounting delivery storage requires review. Retained batches={metrics.batch_count}, table bytes={metrics.table_bytes}.", + ) + if metrics.oldest_pending_seconds > 300: + await _alert( + proxy_logging_obj, + message=f"Accounting deliveries are delayed. Pending={metrics.pending_count}, oldest={metrics.oldest_pending_seconds:.0f}s.", + ) + finally: + if acquired: + await writer.pod_lock_manager.release_lock(cronjob_id=_JOB) + + +async def _alert(alerts: _AccountingAlerts, message: str) -> None: + await alerts.alerting_handler(message=message, level="High", alert_type=AlertType.failed_tracking_spend) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index c82d25c9425..af084a464ee 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -620,7 +620,7 @@ class DualCache(BaseCache): increment_list: list["RedisPipelineIncrementOperation"], local_only: bool = False, parent_otel_span: Span | None = None, - **kwargs, + **kwargs: object, # kwargs-ok: accepted and unused, as the cache interface's other calls take them ) -> list[float] | None: result: list[float] | None = None try: diff --git a/litellm/constants.py b/litellm/constants.py index 32b47bc6b46..50e090eabaf 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1673,6 +1673,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9154a6fdc30..3910a554905 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -616,11 +616,17 @@ def cost_per_token( ) if per_second is not None: return per_second + if custom_llm_provider == "anthropic": + return anthropic_cost_per_token( + model=model, usage=usage_block, service_tier=service_tier, model_info=requested_model_info_override + ) return generic_cost_per_token( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider or "", service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, model_info=requested_model_info_override, ) @@ -3323,15 +3329,35 @@ def batch_cost_calculator( return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift +def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float | None: + return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None) + + +def _cached_audio_batch_fallback( + model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int +) -> float: + if batch_cache_read_rate is not None: + return batch_cache_read_rate + standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens) + if standard_audio_rate is not None: + return standard_audio_rate + return 0.0 if cache_read_rate is None else cache_read_rate + + def _batch_prompt_cost( usage: Usage, model_info: ModelInfo, input_rate: float, - cache_read_rate: float | None, + batch_cache_read_rate: float | None, cache_creation_rate: float | None, cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: details: Final = parse_prompt_tokens_details(usage) + cache_read_rate: Final = ( + _standard_rate(model_info, "cache_read_input_token_cost", usage.prompt_tokens) + if batch_cache_read_rate is None + else batch_cache_read_rate + ) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) @@ -3352,7 +3378,11 @@ def _batch_prompt_cost( + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + cached_audio_tokens - * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) + * _batch_rate( + model_info, + "cache_read_input_audio_token_cost_batches", + _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens), + ) + ( calculate_cache_writing_cost( written_tokens, diff --git a/litellm/integrations/email_templates/model_deprecation_email.py b/litellm/integrations/email_templates/model_deprecation_email.py new file mode 100644 index 00000000000..5d325b2442a --- /dev/null +++ b/litellm/integrations/email_templates/model_deprecation_email.py @@ -0,0 +1,181 @@ +"""Model deprecation emails with provider identity, a migration comparison and next steps.""" + +from collections.abc import Mapping +from html import escape +from types import MappingProxyType +from typing import Final +from urllib.parse import urlencode, urlsplit + +from litellm.types.integrations.model_deprecation_email import ( + DeprecationEmailMilestone, + ModelDeprecationEmail, + RenderedDeprecationEmail, +) +from litellm.types.proxy.model_deprecation import SuccessorModelSource + +_LOGO_BASE_URL: Final = ( + "https://raw.githubusercontent.com/BerriAI/litellm/main/ui/litellm-dashboard/public/assets/logos" +) +_EMAIL_LOGO_URL: Final = f"{_LOGO_BASE_URL}/litellm_logo.png" +# Reuse dashboard brand assets; PNGs also render in email clients that block SVG. +_PROVIDER_BRANDS: Final[Mapping[str, tuple[str, str]]] = MappingProxyType( + { + "gemini": ("Google", "google_pse.png"), + "vertex_ai": ("Google", "google_pse.png"), + "openai": ("OpenAI", "openai.png"), + "anthropic": ("Anthropic", "anthropic.png"), + "azure": ("Microsoft Azure", "microsoft_azure.png"), + "bedrock": ("AWS Bedrock", "aws.png"), + "bedrock_converse": ("AWS Bedrock", "aws.png"), + } +) + + +def _recommendation_source(source: SuccessorModelSource | None) -> str: + if source is SuccessorModelSource.DEPLOYMENT_CONFIG: + return "Gateway configuration" + if source is SuccessorModelSource.MODEL_COST_MAP: + return "LiteLLM model cost map" + return "Not provided" + + +def _http_url(value: str | None) -> str | None: + if value is None: + return None + try: + parsed: Final = urlsplit(value) + except ValueError: + return None + return value if parsed.scheme in ("http", "https") and parsed.netloc and not parsed.username else None + + +def _provider(model: str, fallback: str | None = None) -> str: + prefix, separator, _ = model.partition("/") + return prefix if separator else fallback or "" + + +def _provider_name(provider: str) -> str: + brand: Final = _PROVIDER_BRANDS.get(provider) + if brand: + return brand[0] + return provider.replace("_", " ") or "The provider" + + +def _model_html(model: str, provider: str) -> str: + brand: Final = _PROVIDER_BRANDS.get(provider) + logo: Final = ( + f'{escape(brand[0], quote=True)}' + if brand + else "" + ) + return ( + f'{logo}' + f'
{escape(_provider_name(provider))}
' + f"
{escape(model)}
" + ) + + +def render_model_deprecation_email(email: ModelDeprecationEmail, *, base_url: str) -> RenderedDeprecationEmail: + gateway: Final = _http_url(base_url) + if gateway is None: + raise ValueError("Model deprecation emails require a public HTTP(S) PROXY_BASE_URL") + urgent: Final = email.milestone is DeprecationEmailMilestone.ONE_DAY + days: Final = int(email.milestone) + timing: Final = "tomorrow" if urgent else "in 30 days" + provider: Final = _provider(email.usage_model, email.model.litellm_provider) + provider_name: Final = _provider_name(provider) + subject: Final = f"[LiteLLM] Model being deprecated by {provider_name} {timing}: {email.usage_model}" + deadline: Final = email.model.deprecation_date.strftime("%d %b %Y") + announcement: Final = f"{provider_name} is deprecating this model on {deadline} (UTC)." + preheader: Final = f"{announcement} Review the model change and migrate your affected keys." + query: Final = urlencode( + { + "tab": "models", + "from": email.start_date.isoformat(), + "to": email.end_date.isoformat(), + } + ) + review_url: Final = f"{gateway.rstrip('/')}/ui/usage?{query}" + replacement: Final = email.model.successor_model + after: Final = ( + _model_html(replacement, _provider(replacement, provider)) + if replacement + else '

No recommended model listed. Choose a supported replacement.

' + ) + source: Final = _recommendation_source(email.model.successor_model_source) + notice: Final = _http_url(email.model.deprecation_source) + source_link: Final = ( + f'Read the provider notice ↗' + if notice + else "" + ) + team_names: Final = ", ".join(email.team_names[:3]) or "Personal keys" + more_teams: Final = f" + {len(email.team_names) - 3} more" if len(email.team_names) > 3 else "" + day_label: Final = "DAY" if urgent else "DAYS" + label: Final = "FINAL REMINDER" if urgent else "DEPRECATION NOTICE" + test_step: Final = ( + "Test the recommended model with your workload." + if replacement + else "Choose a supported replacement and test it with your workload." + ) + switch_step: Final = ( + f"Switch your app or gateway routing before {deadline}. Then check that requests to the old model reach zero." + ) + html: Final = f""" + +{escape(subject)} + + +
{escape(preheader)}
+
+ + + + + +
+ +
LiteLLM/ MODEL DEPRECATIONS
+ +
{label}{days:02d} {day_label} UNTIL DEPRECATION
+

Model being deprecated

+

{escape(announcement)}

+
+ + +
+
BEFORE · MODEL TO REPLACE
+{_model_html(email.usage_model, provider)} +		 +
AFTER · RECOMMENDED MODEL
+{after} +
+

Recommendation source: {escape(source)}
{source_link}

+

Your affected usage

+ + +
{email.key_count:,}
Affected keys
{email.requests:,}
Requests · last 30 days
+

{escape(team_names + more_teams)}
Your keys and teams you administer.

+

What to do

+
    +
  1. Review the affected keys in your usage dashboard.
    2. +
  2. {escape(test_step)}
    3. +
  3. {escape(switch_step)}
+
+Review affected usage ↗
+
+Your gateway admin enabled these emails. Reminder schedule: 30 days and 1 day before deprecation.

LiteLLM · AI Gateway
""" + text: Final = ( + f"{subject}\n\n{announcement}\n\n" + f"BEFORE - MODEL TO REPLACE\n{email.usage_model}\n\n" + f"AFTER - RECOMMENDED MODEL\n{replacement or 'No recommended model listed. Choose a supported replacement.'}\n" + f"Recommendation source: {source}\nProvider notice: {notice or 'Not provided'}\n\n" + f"YOUR AFFECTED USAGE\n{email.key_count} keys | {email.requests} requests in the last 30 days\n" + f"{team_names}{more_teams}\nYour keys and teams you administer.\n\n" + f"WHAT TO DO\n1. Review the affected keys in your usage dashboard.\n2. {test_step}\n3. {switch_step}\n\n" + f"Review affected usage: {review_url}\n\n" + "Your gateway admin enabled these emails. Reminder schedule: 30 days and 1 day before deprecation." + ) + return RenderedDeprecationEmail(subject=subject, html=html, text=text) diff --git a/litellm/integrations/model_deprecation_email.py b/litellm/integrations/model_deprecation_email.py new file mode 100644 index 00000000000..4093a108dcc --- /dev/null +++ b/litellm/integrations/model_deprecation_email.py @@ -0,0 +1,267 @@ +"""Two owner-scoped retirement reminders, with durable per-recipient delivery receipts.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from enum import Enum +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter + +from litellm.integrations.email_templates.model_deprecation_email import ( + render_model_deprecation_email, +) +from litellm.models.team import LiteLLM_TeamTable +from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations +from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.table_repositories import DailyUserSpendRepository +from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.user_repository import UserRepository +from litellm.repositories.verification_token_repository import ( + VerificationTokenRepository, +) +from litellm.types.integrations.model_deprecation_email import ( + MODEL_DEPRECATION_EMAIL_EVENT, + DeprecationEmailMilestone, + ModelDeprecationEmail, +) +from litellm.types.proxy.model_deprecation import ( + DEPRECATION_IDLE_POLL_SECONDS, + ModelDeprecationInfo, +) + +if TYPE_CHECKING: + from litellm import Router + from litellm.proxy.utils import PrismaClient + +_RECEIPT_PREFIX: Final = "model_deprecation_email:" +_PAGE_SIZE: Final = 100 + + +class _OptIn(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + enabled: bool = Field( + False, + strict=True, + validation_alias=AliasPath("email_settings", MODEL_DEPRECATION_EMAIL_EVENT), + ) + + +class _RequestCount(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + api_requests: int + + +class _UsedKey(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + api_key: str + metrics: _RequestCount = Field(alias="_sum") + + +_USED_KEYS: Final = TypeAdapter(tuple[_UsedKey, ...]) + + +class _DeliveryStatus(str, Enum): + SENDING = "sending" + SENT = "sent" + + +class _Receipt(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + revision: int = 0 + status: _DeliveryStatus = _DeliveryStatus.SENDING + lease_until: datetime + + +@dataclass(frozen=True, slots=True) +class _Impact: + key_count: int = 0 + requests: int = 0 + teams: frozenset[str] = frozenset() + + +def _admin_ids(team: LiteLLM_TeamTable | None) -> frozenset[str]: + if team is None: + return frozenset() + return frozenset( + member.user_id for member in (team.members_with_roles or ()) if member.role == "admin" and member.user_id + ) + + +def _recipient_emails( + user_id: str | None, team: LiteLLM_TeamTable | None, addresses: Mapping[str, str] +) -> frozenset[str]: + owner_ids: Final[frozenset[str]] = frozenset((user_id,)) if user_id else frozenset() + resolved: Final = frozenset(addresses[owner] for owner in owner_ids | _admin_ids(team) if owner in addresses) + if team is None: + return resolved + return resolved | frozenset( + member.user_email.strip() + for member in (team.members_with_roles or ()) + if member.role == "admin" + and member.user_email + and member.user_email.strip() + and member.user_id not in addresses + ) + + +def _usage_model(model: ModelDeprecationInfo) -> str: + name: Final = model.litellm_model or model.model_name + provider: Final = model.litellm_provider + if provider and not name.startswith(f"{provider}/"): + return f"{provider}/{name}" + return name + + +def _receipt_key(email: ModelDeprecationEmail) -> str: + identity: Final = json.dumps( + [ + email.usage_model, + email.model.deprecation_date.isoformat(), + int(email.milestone), + email.recipient_email.casefold(), + ] + ) + return _RECEIPT_PREFIX + hashlib.sha256(identity.encode()).hexdigest() + + +class ModelDeprecationEmailSender: + def __init__(self, prisma_client: PrismaClient) -> None: + self.prisma: Final = prisma_client + self.config: Final = ConfigRepository(prisma_client, use_writer=True) + + async def _impacts(self, model: str, start: date, end: date) -> Mapping[str, _Impact]: + impacts: Final[dict[str, _Impact]] = {} # mutable-ok: combines per-recipient counts across bounded usage pages + offset: int = 0 # rebind-ok: each page advances the stable key grouping cursor + while True: + raw = await DailyUserSpendRepository(self.prisma).table.group_by( + by=["api_key"], + where={ + "model": model, + "date": {"gte": start.isoformat(), "lte": end.isoformat()}, + "api_requests": {"gt": 0}, + }, + sum={"api_requests": True}, + order={"api_key": "asc"}, + skip=offset, + take=_PAGE_SIZE, + ) + usage = _USED_KEYS.validate_python(raw) + if not usage: + return MappingProxyType(impacts) + keys = await VerificationTokenRepository(self.prisma).find_many_in("token", (row.api_key for row in usage)) + teams = await TeamRepository(self.prisma).find_many_in( + "team_id", frozenset(key.team_id for key in keys if key.team_id) + ) + team_by_id = {team.team_id: team for team in teams} + ids = frozenset(key.user_id for key in keys if key.user_id) | frozenset( + chain.from_iterable(_admin_ids(team) for team in teams) + ) + users = await UserRepository(self.prisma).find_many_in("user_id", ids) + addresses = { + user.user_id: user.user_email.strip() for user in users if user.user_email and user.user_email.strip() + } + counts = {row.api_key: row.metrics.api_requests for row in usage} + for key in keys: + team = team_by_id.get(key.team_id) if key.team_id else None + if key.token is None: + continue + recipients = _recipient_emails(key.user_id, team, addresses) + for address in recipients: + previous = impacts.get(address, _Impact()) + names: frozenset[str] = frozenset((team.team_alias or team.team_id,)) if team else frozenset() + impacts[address] = _Impact( + key_count=previous.key_count + 1, + requests=previous.requests + counts[key.token], + teams=previous.teams | names, + ) + if len(usage) < _PAGE_SIZE: + return MappingProxyType(impacts) + offset += _PAGE_SIZE + + async def _deliver(self, email: ModelDeprecationEmail, now: datetime, base_url: str) -> bool: + from litellm.proxy.utils import send_email + + key: Final = _receipt_key(email) + record: Final = await self.config.get_param(key) + previous: Final = _Receipt.model_validate(record.param_value) if record else None + if previous and (previous.status is _DeliveryStatus.SENT or previous.lease_until > now): + return False + revision: Final = previous.revision if previous else 0 + claim: Final = _Receipt(revision=revision + 1, lease_until=now + timedelta(minutes=5)) + if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), revision): + return False + content: Final = render_model_deprecation_email(email, base_url=base_url) + if not await send_email( + receiver_email=email.recipient_email, + subject=content.subject, + html=content.html, + text=content.text, + message_id=f"", + ): + return False + sent: Final = claim.model_copy(update={"revision": claim.revision + 1, "status": _DeliveryStatus.SENT}) + if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision): + raise RuntimeError("Could not record the delivered model deprecation reminder") + return True + + async def run(self, llm_router: Router, *, now: datetime | None = None) -> int: + current: Final = now or datetime.now(timezone.utc) + settings: Final = await self.config.get_param("general_settings") + if settings is None or not _OptIn.model_validate(settings.param_value).enabled: + return 0 + base_url: Final = os.getenv("PROXY_BASE_URL", "") + if not base_url: + return 0 + await self.config.delete_params_older_than(_RECEIPT_PREFIX, current - timedelta(days=60)) + snapshot: Final = collect_model_deprecations(llm_router=llm_router, today=current.date()) + start: Final = current.date() - timedelta(days=29) + seen: Final[set[tuple[str, date]]] = set() # mutable-ok: deduplicates aliases in this pass + sent: int = 0 # rebind-ok: counts successfully delivered reminders in this pass + for model in snapshot.imminent: + if model.days_until_deprecation not in tuple(DeprecationEmailMilestone): + continue + name = _usage_model(model) + identity = (name, model.deprecation_date) + if identity in seen: + continue + seen.add(identity) + impacts = await self._impacts(name, start, current.date()) + for address, impact in sorted(impacts.items()): + email = ModelDeprecationEmail( + model=model, + usage_model=name, + recipient_email=address, + milestone=DeprecationEmailMilestone(model.days_until_deprecation), + key_count=impact.key_count, + requests=impact.requests, + team_names=tuple(sorted(impact.teams)), + start_date=start, + end_date=current.date(), + ) + if await self._deliver(email, now or datetime.now(timezone.utc), base_url): + sent += 1 + return sent + + +async def run_scheduled_model_deprecation_emails() -> None: + """Poll the email opt-in independently; a large owner mailing never delays Slack checks.""" + from litellm._logging import verbose_proxy_logger + + while True: + try: + from litellm.proxy.proxy_server import llm_router, prisma_client + + if llm_router is not None and prisma_client is not None: + await ModelDeprecationEmailSender(prisma_client).run(llm_router) + except Exception: # noqa: BLE001 # transient SMTP/DB failures must not terminate the reminder loop + verbose_proxy_logger.exception("Error sending model deprecation email reminders") + await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index ece85b09432..0f98a8fce0e 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -338,10 +338,12 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_ToolTable", "LiteLLM_TelemetryReport", "LiteLLM_AccessGroupTable", - "LiteLLM_AccountingDelivery", - "LiteLLM_AccountingEvent", + "LiteLLM_SpendLogDelivery", "LiteLLM_ClaudeCodePluginTable", "LiteLLM_ClaudeCodePluginUpload", + "LiteLLM_ClaudeCodePluginVersion", + "LiteLLM_SkillSubmission", + "LiteLLM_ClaudeCodePluginDownload", "LiteLLM_MemoryTable", "LiteLLM_AdaptiveRouterState", "LiteLLM_AdaptiveRouterSession", diff --git a/litellm/litellm_core_utils/add_retry_fallback_headers.py b/litellm/litellm_core_utils/add_retry_fallback_headers.py index 12b26954e4c..4cf2d1d58d8 100644 --- a/litellm/litellm_core_utils/add_retry_fallback_headers.py +++ b/litellm/litellm_core_utils/add_retry_fallback_headers.py @@ -202,7 +202,7 @@ def response_total_token_count(response: object) -> int: def apply_remaining_usage_headers( additional_headers: dict[str, object], - remaining_usage: dict[str, int], + remaining_usage: Mapping[str, object], ) -> None: for header, value in remaining_usage.items(): if value is not None and header not in additional_headers: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 43c586e7245..b28d5a4ea21 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -108,6 +108,17 @@ def as_object_list(value: object) -> Sequence[object] | None: return value if isinstance(value, list) else None +_Key = TypeVar("_Key") # rebind-ok: a TypeVar cannot be Final +_Value = TypeVar("_Value") # rebind-ok: a TypeVar cannot be Final + + +def without( + mapping: Mapping[_Key, _Value], *keys: object +) -> dict[_Key, _Value]: # mutable-ok: a fresh copy the caller owns + """``mapping`` without ``keys``, as a new dict in the same order; ``mapping`` is unchanged.""" + return {key: value for key, value in mapping.items() if key not in keys} + + def decoded_json_object(value: object) -> Mapping[str, object]: mapping: Final = as_str_mapping(value) if mapping is not None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 1fcb06b80e8..ce57a9bd4a5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7143,6 +7143,7 @@ class StandardLoggingPayloadSetup: init_response_obj: object, api_base: str | None = None, router_model_id: str | None = None, + custom_model_info: ModelInfo | None = None, ) -> StandardLoggingModelInformation: model_cost_name: Final = select_model_name_for_cost_calc( model=base_model if custom_pricing else None, @@ -7161,6 +7162,8 @@ class StandardLoggingPayloadSetup: if router_model_id is not None else model_cost_name ) + if model_cost_name is not None and custom_model_info is not None: + return StandardLoggingModelInformation(model_map_key=model_cost_name, model_map_value=custom_model_info) if model_cost_name is None: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: @@ -7911,6 +7914,7 @@ def get_standard_logging_object_payload( init_response_obj=init_response_obj, api_base=litellm_params.get("api_base"), router_model_id=logging_obj.get_router_model_id(), + custom_model_info=logging_obj.custom_model_info if custom_pricing else None, ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 9a0a35979db..08ab2d1a2e5 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -1081,4 +1081,8 @@ def convert_to_model_response_object( end_time=end_time, convert_tool_call_to_json_mode=convert_tool_call_to_json_mode, ) - raise Exception(f"Invalid response object {traceback.format_exc()}\n\nreceived_args={received_args}") + # The conversion's own traceback, without the errors being handled when it ran: a router + # retry runs inside the handler of the attempt before it, and that attempt's error text + # (a rate limit) would otherwise decide how this one is classified. + own_traceback: Final = "".join(traceback.format_exception(type(e), e, e.__traceback__, chain=False)) + raise Exception(f"Invalid response object {own_traceback}\n\nreceived_args={received_args}") diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index a06860e7786..62bdc1c10a7 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -378,7 +378,8 @@ def filter_team_based_models( ids_to_remove: Final = set() if isinstance(healthy_deployments, dict): - return healthy_deployments + # A deployment asked for by id is filtered like any other: another team's is not served. + return [] if _owner_team(healthy_deployments) not in (None, request_team_id) else healthy_deployments # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # the legacy untyped deployment dict for deployment in healthy_deployments: _model_info = deployment.get("model_info") or {} model_team_id = _model_info.get("team_id") @@ -431,6 +432,16 @@ def _deployment_supports_web_search(deployment: dict) -> bool: return True +def is_web_search_request(request_kwargs: Mapping[str, object]) -> bool: + tools: Final[object] = request_kwargs.get("tools") + return isinstance(tools, (list, tuple)) and any( + isinstance(tool, dict) + and isinstance(tool_type := tool.get("type"), str) + and tool_type in _WEB_SEARCH_TOOL_TYPES + for tool in tools + ) + + def filter_web_search_deployments( healthy_deployments: list[dict] | dict, request_kwargs: dict | None = None, @@ -445,14 +456,7 @@ def filter_web_search_deployments( if isinstance(healthy_deployments, dict): return healthy_deployments - tools: Final[object] = request_kwargs.get("tools") - is_web_search_request: Final = isinstance(tools, (list, tuple)) and any( - isinstance(tool, dict) - and isinstance(tool_type := tool.get("type"), str) - and tool_type in _WEB_SEARCH_TOOL_TYPES - for tool in tools - ) - if not is_web_search_request: + if not is_web_search_request(request_kwargs): return healthy_deployments # Filter out deployments that don't support web search @@ -607,3 +611,9 @@ def warn_on_provider_credential_mismatch(model_name: str, litellm_params: Mappin ) verbose_router_logger.warning(warning) return warning + + +def _owner_team(deployment: Mapping[str, object]) -> object: + """The team a deployment belongs to (``model_info.team_id``), or None.""" + model_info: Final = deployment.get("model_info") + return model_info.get("team_id") if isinstance(model_info, Mapping) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a deployment dict's model_info diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 4eae06dbcdf..c273c715ba6 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -734,6 +734,7 @@ class ModelResponseIterator: self._split_answer_blocks: tuple[str, ...] = () self._withheld_answer_close: str = "" self._streamed_answer_keys: frozenset[str] | None = None + self._held_prose: str = "" # For handling partial JSON chunks from fragmentation # See: https://github.com/BerriAI/litellm/issues/17473 @@ -826,8 +827,13 @@ class ModelResponseIterator: thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = [] if "text" in content_block["delta"]: - text = content_block["delta"]["text"] - return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content + return ( + self._hold_prose(content_block["delta"]["text"]), + tool_use, + thinking_blocks, + provider_specific_fields, + reasoning_content, + ) self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: # Only emit tool calls if we're in a tool_use or server_tool_use block @@ -1025,7 +1031,7 @@ class ModelResponseIterator: if self.current_content_block_type in _THINKING_BLOCK_TYPES: self.thinking_block_index += 1 if content_block_start["content_block"]["type"] == "text": - text = self._release_answer_close() + content_block_start["content_block"]["text"] + text = self._release_answer_close() + self._hold_prose(content_block_start["content_block"]["text"]) elif ( content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use" @@ -1161,7 +1167,7 @@ class ModelResponseIterator: elif type_chunk == "message_delta": finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk) provider_specific_fields.update(stop_reason_fields) - text = self._release_answer_close() + self._drain_split_answer_blocks() + text = self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose() if container: provider_specific_fields["container"] = container # Store container_id and re-emit code_interpreter_results @@ -1211,7 +1217,9 @@ class ModelResponseIterator: status_code=500, # it looks like Anthropic API does not return a status code in the chunk error - default to 500 ) - text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use) + text, tool_use = self._handle_json_mode_chunk( + text=text + self._settle_held_prose(tool_use), tool_use=tool_use + ) refusal_fields: Final = refusal_params(provider_specific_fields) returned_chunk: Final = ModelResponseStream( @@ -1309,6 +1317,24 @@ class ModelResponseIterator: self._response_format_verdict = _envelope_verdict(buffered, schema, RESPONSE_FORMAT_TOOL_NAME) return self._withhold_answer_close(buffered) if self._response_format_verdict == "pass_through" else "" + def _hold_prose(self, text: str) -> str: + if not self.json_mode or self.response_format_schema is None: + return text + self._held_prose += text + return "" + + def _settle_held_prose(self, tool_use: ChatCompletionToolCallChunk | None) -> str: + if tool_use is None or tool_use.get("id") is None or self.current_content_block_type != "tool_use": + return "" + held: Final = self._held_prose + self._held_prose = "" + return "" if tool_use.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME else held + + def _release_held_prose(self) -> str: + held: Final = self._held_prose + self._held_prose = "" + return "" if self.converted_response_format_tool else held + def _withhold_answer_close(self, text: str) -> str: pending: Final = self._withheld_answer_close + text body: Final = pending.rstrip() @@ -1362,7 +1388,9 @@ class ModelResponseIterator: def cut_stream_chunk(self) -> ModelResponseStream | None: unsent: Final = self._flush_response_format_buffer() if self._response_format_buffer.getvalue() else "" - text: Final = unsent + self._release_answer_close() + self._drain_split_answer_blocks() + text: Final = ( + unsent + self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose() + ) if not text: return None return ModelResponseStream( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 09d16181c3a..4f890ed922f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -26,6 +26,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.asyncify import is_event_loop_running from litellm.litellm_core_utils.core_helpers import ( @@ -1786,7 +1787,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - open_object_takes_forced_json_tool: Final = ( + open_object_takes_json_tool: Final = ( json_schema is not None and schema_has_open_object(json_schema) and not self.is_thinking_requested(request_params) @@ -1794,7 +1795,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and not request_params.get("tools") and not AnthropicModelInfo.forced_tool_use_unsupported(model) ) - return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool + return self.uses_native_structured_output(model) and not open_object_takes_json_tool def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema))) @@ -1809,7 +1810,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema)) def map_response_format_to_anthropic_tool( - self, value: dict | None, optional_params: dict, is_thinking_enabled: bool + self, value: dict | None, optional_params: dict, tool_is_forced: bool ) -> AnthropicMessagesTool | None: ignore_response_format_types: Final = ["text"] if value is None or value["type"] in ignore_response_format_types: # value is a no-op @@ -1827,6 +1828,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, + description=None if tool_is_forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) return _tool @@ -1929,7 +1931,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drop_params=drop_params, runs_hosted_mcp=self.runs_hosted_mcp, ) - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params) + response_format_tool_is_forced: Final = ( + "response_format" in request_params + and AnthropicModelInfo.response_format_tool_can_be_forced( + model, self._resolved_provider, REQUEST_PARAMS.validate_python(request_params) + ) + ) # NB: ``map_openai_params`` deliberately does NOT sanitize tool names # here. Names are the *original* OpenAI names at this stage, and must @@ -1989,10 +1996,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if _output_format is not None: optional_params["output_format"] = _output_format else: - _tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) + _tool = self.map_response_format_to_anthropic_tool( + value, optional_params, response_format_tool_is_forced + ) if _tool is None: continue - if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model): + if response_format_tool_is_forced: _tool_choice = { "name": RESPONSE_FORMAT_TOOL_NAME, "type": "tool", @@ -2121,12 +2130,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def _create_json_tool_call_for_response_format( self, json_schema: dict | None = None, + description: str | None = None, ) -> AnthropicMessagesTool: """ Handles creating a tool call for getting responses in JSON format. Args: json_schema (Optional[dict]): The JSON schema the response should be in + description (Optional[str]): The tool description, left off the tool when None Returns: AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format @@ -2143,11 +2154,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: _input_schema.update(cast(AnthropicInputSchema, json_schema)) - _tool: Final = AnthropicMessagesTool( - name=RESPONSE_FORMAT_TOOL_NAME, - input_schema=_input_schema, + if description is None: + return AnthropicMessagesTool(name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema) + return AnthropicMessagesTool( + name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema, description=description ) - return _tool def should_strip_billing_metadata(self) -> bool: """ diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bfd1341711a..bc9bd769d83 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1141,6 +1141,34 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider) + @staticmethod + def thinks_by_default(model: str, custom_llm_provider: str) -> bool: + return AnthropicModelInfo._is_always_on_thinking_model( + model, custom_llm_provider + ) or AnthropicModelInfo._supports_model_capability(model, "thinking_on_by_default", custom_llm_provider) + + @staticmethod + def will_think(model: str, custom_llm_provider: str, request_params: Mapping[str, object]) -> bool: + if AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider): + return True + match request_params: + case {"thinking": True | {"type": "enabled" | "adaptive"}}: + return True + case {"reasoning_effort": effort} if effort is not None and effort != "none": + return True + case {"thinking": {"type": "disabled"}} | {"reasoning_effort": "none"}: + return False + case _: + return AnthropicModelInfo.thinks_by_default(model, custom_llm_provider) + + @staticmethod + def response_format_tool_can_be_forced( + model: str, custom_llm_provider: str, request_params: Mapping[str, object] + ) -> bool: + return not AnthropicModelInfo.forced_tool_use_unsupported(model) and not AnthropicModelInfo.will_think( + model, custom_llm_provider, request_params + ) + @staticmethod def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool: """Whether ``model`` is an adaptive-thinking model that still accepts legacy diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index c41deb00f1d..87be996ebbf 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -670,6 +670,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage + # The provider's error behind this stream's error frame, for a router that accounts for it. + self.stream_failure: Exception | None = None self._refusal_text: str = "" self._paused_turn: bool = False self._claude_refusal_stop_details: AnthropicStopDetails | None = None @@ -1396,6 +1398,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): verbose_logger.exception("Anthropic Adapter - mid-stream error: %s", e) logging_obj: Final = self.litellm_logging_obj provider_error: Final = _provider_error(e) + self.stream_failure = provider_error if logging_obj is not None and logging_obj.on_detached_stream_failure is not None: if provider_error is e: raise @@ -1619,6 +1622,10 @@ class AnthropicSSEStream(AsyncIterator[bytes]): self._byte_stream: Final[AsyncIterator[bytes]] = anthropic_wrapper.async_anthropic_sse_wrapper() self._hidden_params: dict[str, object] = {} + @property + def stream_failure(self) -> Exception | None: + return self._anthropic_wrapper.stream_failure + @property def chunks(self) -> "list[ModelResponseStream] | None": return self._anthropic_wrapper.chunks diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index dae96ba4ce1..f0c3316c509 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -141,6 +141,8 @@ class AnthropicResponsesStreamWrapper: if litellm_logging_obj is not None: litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id) self._unreported_failure: Exception | None = None + # The failure behind this stream's error frame, for a router that accounts for it. + self.stream_failure: Exception | None = None self._current_block_index: int = -1 # Map item_id -> content_block_index so we can stop the right block later self._item_id_to_block_index: dict[str, int] = {} @@ -467,6 +469,7 @@ class AnthropicResponsesStreamWrapper: llm_provider=getattr(self.responses_stream, "custom_llm_provider", None), model=self.model, ) + self.stream_failure = self._unreported_failure self._chunk_queue.append(_anthropic_error_chunk(status_code, message)) async def _report_unreported_failure(self) -> None: @@ -578,5 +581,9 @@ class AnthropicResponsesSSEByteStream: async def aclose(self) -> None: await self._frames.aclose() + @property + def stream_failure(self) -> Exception | None: + return self._wrapper.stream_failure + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": return self._wrapper.partial_response_on_disconnect() diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 6c954d00e64..f95dc1e4388 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -3,11 +3,10 @@ from collections.abc import Callable, Mapping from copy import deepcopy from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, cast from urllib.parse import urlsplit import httpx -from openai.types.responses import ResponseReasoningItem from pydantic import TypeAdapter from litellm._logging import verbose_logger @@ -16,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cach from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * @@ -72,6 +72,21 @@ def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[st return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) +_AZURE_REJECTED_INPUT_FIELDS: Final = MappingProxyType( + {"message": frozenset({"status"}), "reasoning": frozenset({"status"})} +) +_AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS: Final = MappingProxyType( + {**_AZURE_REJECTED_INPUT_FIELDS, "reasoning": frozenset({"status", "content"})} +) + + +def _without_fields_azure_rejects(item: object, rejected_fields: Mapping[str, frozenset[str]]) -> object: + if not is_str_keyed_mapping(item): + return item + rejected: Final = rejected_fields.get(str(item.get("type")), frozenset()) + return {key: value for key, value in item.items() if key not in rejected} if rejected else item + + class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Parameters not supported by Azure Responses API AZURE_UNSUPPORTED_PARAMS = ["context_management"] @@ -122,68 +137,17 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def get_stripped_model_name(self, model: str) -> str: return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "") - def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]: - """ - Handle reasoning items to filter out the status field. - Issue: https://github.com/BerriAI/litellm/issues/13484 - - Azure OpenAI API does not accept 'status' field in reasoning input items. - """ - if item.get("type") == "reasoning": - try: - # Ensure required fields are present for ResponseReasoningItem - item_data: Final = dict(item) - if "summary" not in item_data: - item_data["summary"] = ( - item_data.get("reasoning_content", "")[:100] + "..." - if len(item_data.get("reasoning_content", "")) > 100 - else item_data.get("reasoning_content", "") - ) - - # Create ResponseReasoningItem object from the item data - reasoning_item: Final = ResponseReasoningItem.model_validate(item_data) - - # Convert back to dict with exclude_none=True to exclude None fields - dict_reasoning_item: Final = reasoning_item.model_dump(exclude_none=True) - dict_reasoning_item.pop("status", None) - - return dict_reasoning_item - except Exception as e: - verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e) - # Fallback: manually filter out known None fields - filtered_item: Final = { - k: v - for k, v in item.items() - if v is not None or k not in {"status", "content", "encrypted_content"} - } - return filtered_item - return item - def _validate_input_param( - self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False + self, input: str | ResponseInputParam, *, drop_params: bool = False ) -> str | ResponseInputParam: - """ - Override parent method to also filter out 'status' field from message items. - Azure OpenAI API does not accept 'status' field in input messages. - """ - from typing import cast - - # First call parent's validation - validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by) - - # Then filter out status from message items - if isinstance(validated_input, list): - filtered_input: Final[list[object]] = [] - for item in validated_input: - if isinstance(item, dict) and item.get("type") == "message": - # Filter out status field from message items - filtered_item = {k: v for k, v in item.items() if k != "status"} - filtered_input.append(filtered_item) - else: - filtered_input.append(item) - return cast(ResponseInputParam, filtered_input) - - return validated_input + validated_input: Final = super()._validate_input_param(input, drop_params=drop_params) + if not isinstance(validated_input, list): + return validated_input + rejected_fields: Final = ( + _AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS if drop_params else _AZURE_REJECTED_INPUT_FIELDS + ) + sanitized_items: Final = [_without_fields_azure_rejects(item, rejected_fields) for item in validated_input] + return cast("ResponseInputParam", sanitized_items) # cast-ok: items keep their shape minus rejected fields def transform_responses_api_request( self, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 056f34f1bf4..03fbc35a97f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -22,6 +22,7 @@ from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_hea from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.integrations.anthropic_cache_control_hook import ( MAX_CACHE_CONTROL_BLOCKS, @@ -950,19 +951,24 @@ class AmazonConverseConfig(BaseConfig): def _supports_native_structured_outputs( model: str, custom_llm_provider: str | None, + will_think: bool, thinking_requested: bool, model_info: Mapping[str, object] | None, ) -> bool: """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and - ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the - request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot - force the json tool gets it unless that key is false. + ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude + will think, which the json tool path would drop. When the request does not ask for thinking, a model + that cannot force the json tool gets it unless that key is false. Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html """ return bedrock_chat_uses_native_structured_output( - model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info + model, + will_think=will_think, + thinking_requested=thinking_requested, + custom_llm_provider=custom_llm_provider or "bedrock_converse", + model_info=model_info, ) @staticmethod @@ -1005,11 +1011,13 @@ class AmazonConverseConfig(BaseConfig): self, model: str, json_schema: Mapping[str, object], + *, + will_think: bool, thinking_requested: bool, model_info: Mapping[str, object] | None, ) -> Mapping[str, object] | None: if not self._supports_native_structured_outputs( - model, self.custom_llm_provider, thinking_requested, model_info + model, self.custom_llm_provider, will_think, thinking_requested, model_info ): return None if BedrockModelInfo.get_base_model(model).startswith("anthropic"): @@ -1095,7 +1103,6 @@ class AmazonConverseConfig(BaseConfig): request_params: Final = tool_params_without_hosted_mcp( non_default_params, model=model, route="bedrock", drop_params=drop_params ) - is_thinking_enabled: Final = self.is_thinking_enabled(request_params) base_model: Final = BedrockModelInfo.get_base_model(model) drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model) drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param( @@ -1109,7 +1116,6 @@ class AmazonConverseConfig(BaseConfig): model=model, optional_params=optional_params, non_default_params=request_params, - is_thinking_enabled=is_thinking_enabled, model_info=model_info, ) if param == "max_tokens" or param == "max_completion_tokens": @@ -1314,7 +1320,6 @@ class AmazonConverseConfig(BaseConfig): model: str, optional_params: dict, non_default_params: dict, - is_thinking_enabled: bool, model_info: Mapping[str, object] | None = None, ) -> dict: """ @@ -1339,9 +1344,16 @@ class AmazonConverseConfig(BaseConfig): if "type" in value and value["type"] == "text": return optional_params - thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)) + request_params: Final = REQUEST_PARAMS.validate_python(non_default_params) + thinking_requested: Final = self.is_thinking_requested(request_params) native_schema: Final = ( - self._native_output_schema(model, json_schema, thinking_requested, model_info) + self._native_output_schema( + model, + json_schema, + will_think=AnthropicModelInfo.will_think(model, "bedrock", request_params), + thinking_requested=thinking_requested, + model_info=model_info, + ) if json_schema is not None else None ) @@ -1359,17 +1371,24 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_is_forced: Final = litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) and AnthropicModelInfo.response_format_tool_can_be_forced(model, "bedrock", request_params) + schema_description: Final = description if isinstance(description, str) else None + instructs_unforced_tool: Final = not tool_is_forced and BedrockModelInfo.get_base_model(model).startswith( + "anthropic" + ) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, schema_description))) + if instructs_unforced_tool + else schema_description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) - if ( - litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) - ): + if tool_is_forced: optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) ) @@ -1380,8 +1399,8 @@ class AmazonConverseConfig(BaseConfig): # returns {} instead of the requested JSON. The model already # returns JSON when the prompt asks for it. - # A request that goes native only because the model can't force the json tool keeps the json tool's faked - # stream, so it still needs only bedrock:InvokeModel. + # A request that goes native without asking for thinking keeps the json tool's faked stream, so it still + # needs only bedrock:InvokeModel, unless the model takes native output on every request. streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output( model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info ) @@ -2219,8 +2238,8 @@ class AmazonConverseConfig(BaseConfig): } if additional_request_params: data["additionalModelRequestFields"] = additional_request_params - if "thinking" in additional_request_params: - data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",) + if "thinking" in additional_request_params or AnthropicModelInfo.thinks_by_default(model, "bedrock"): + data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",) if system_content_blocks: data["system"] = system_content_blocks diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index dc85028440f..62e1bee5217 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -142,14 +142,19 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - return self.chat_uses_native_structured_output( - model, self.is_thinking_requested(request_params), self._deployment_model_info - ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + return self.chat_uses_native_structured_output(model, request_params, self._deployment_model_info) and ( + json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None + ) def chat_uses_native_structured_output( - self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info) + return bedrock_chat_uses_native_structured_output( + model, + will_think=self.will_think(model, "bedrock", request_params), + thinking_requested=self.is_thinking_requested(request_params), + model_info=model_info, + ) def uses_opt_in_native_output( self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] @@ -359,7 +364,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): request_body=anthropic_request, model_info=model_info, chat_uses_native_output=lambda: self.chat_uses_native_structured_output( - model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info + model, _NON_DEFAULT_PARAMS.validate_python(filtered_params), model_info ), ) strip_unsupported_bedrock_invoke_output_config_keys( diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index 952ca7a2cd4..7e635a61d4d 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -55,9 +55,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): ) def chat_uses_native_structured_output( - self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info) + return bedrock_chat_opts_into_native_structured_output( + model, self.will_think(model, "bedrock", request_params), model_info=model_info + ) def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]: return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params)) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index c1b3b5616b8..dfee28a7cc9 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -356,13 +356,15 @@ def bedrock_chat_opts_into_native_structured_output( def bedrock_chat_uses_native_structured_output( model: str, + *, + will_think: bool, thinking_requested: bool, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> bool: - return bedrock_chat_opts_into_native_structured_output( - model, thinking_requested, custom_llm_provider, model_info - ) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info)) + return bedrock_chat_opts_into_native_structured_output(model, will_think, custom_llm_provider, model_info) or ( + not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info) + ) def bedrock_native_output_is_deployment_opt_in( diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index d943718ed73..859211bd40d 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -31,6 +31,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict ) +from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.databricks import ( AllDatabricksContentValues, @@ -541,12 +542,12 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): model: str, value: dict | None, optional_params: dict, - is_thinking_enabled: bool, + tool_is_forced: bool, ) -> DatabricksTool | None: if value is None: return None - tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) + tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, tool_is_forced) databricks_tool: Final = self.convert_anthropic_tool_to_databricks_tool(tool) return databricks_tool @@ -579,7 +580,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): if "claude" in model else non_default_params ) - is_thinking_enabled: Final = self.is_thinking_enabled(request_params) mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params) if "claude" in model: AnthropicConfig.translate_legacy_thinking_for_adaptive_model( @@ -596,17 +596,20 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): mapped_params.pop("max_completion_tokens", None) if "response_format" in request_params and "claude" in model: + tool_is_forced: Final = AnthropicConfig.response_format_tool_can_be_forced( + model, "databricks", REQUEST_PARAMS.validate_python(request_params) + ) _tool: Final = self.map_response_format_to_databricks_tool( model, request_params["response_format"], mapped_params, - is_thinking_enabled, + tool_is_forced, ) if _tool is not None: self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) optional_params["json_mode"] = True - if not is_thinking_enabled: + if tool_is_forced: _tool_choice: Final = ChatCompletionToolChoiceObjectParam( type="function", function=ChatCompletionToolChoiceFunctionParam(name=RESPONSE_FORMAT_TOOL_NAME), diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 1ecef46ffb1..61d237de2d8 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -6,6 +6,7 @@ Why separate file? Make it easy to see how transformation works Docs - https://docs.mistral.ai/api/ """ +import reprlib from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload @@ -45,7 +46,13 @@ from litellm.types.llms.mistral import ( MistralThinkingBlock, MistralToolCallMessage, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionTextObject, + ChatCompletionToolChoiceFunctionParam, + ChatCompletionToolChoiceObjectParam, +) from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -79,6 +86,10 @@ _REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system pr _NON_LEVEL_EFFORT_REASON: Final = ( "the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort." ) +_UNUSABLE_TOOL_CHOICE_REASON: Final = ( + "the Mistral API takes tool_choice only as 'auto', 'none', 'any', 'required', " + "or {'type': 'function', 'function': {'name': }}." +) _DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"}) _REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"}) @@ -118,7 +129,7 @@ class MistralConfig(OpenAIGPTConfig): - `tools` (list or null): A list of available tools for the model. Use this to specify functions for which the model can generate JSON inputs. - - `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'. + - `tool_choice` (string - 'auto'/'any'/'none', an object naming one function such as {"type": "function", "function": {"name": "my_function"}}, or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'. - `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array @@ -135,7 +146,7 @@ class MistralConfig(OpenAIGPTConfig): top_p: int | None = None max_tokens: int | None = None tools: list | None = None - tool_choice: Literal["auto", "any", "none"] | None = None + tool_choice: Literal["auto", "any", "none"] | ChatCompletionToolChoiceObjectParam | None = None random_seed: int | None = None safe_prompt: bool | None = None response_format: dict | None = None @@ -147,7 +158,7 @@ class MistralConfig(OpenAIGPTConfig): top_p: int | None = None, max_tokens: int | None = None, tools: list | None = None, - tool_choice: Literal["auto", "any", "none"] | None = None, + tool_choice: Literal["auto", "any", "none"] | ChatCompletionToolChoiceObjectParam | None = None, random_seed: int | None = None, safe_prompt: bool | None = None, response_format: dict | None = None, @@ -192,13 +203,40 @@ class MistralConfig(OpenAIGPTConfig): *(("prompt_cache_key",) if accepts_prompt_cache_key else ()), ] - def _map_tool_choice(self, tool_choice: str) -> str: - if tool_choice == "auto" or tool_choice == "none": - return tool_choice - elif tool_choice == "required": - return "any" - else: # openai 'tool_choice' object param not supported by Mistral API - return "any" + @staticmethod + def _map_tool_choice(tool_choice: object) -> str | ChatCompletionToolChoiceObjectParam | None: + match tool_choice: + case str(choice) if choice in ("auto", "none"): + return choice + case "required" | "any": + return "any" + case {"type": "function", "function": {"name": str(name)}} if name: + return ChatCompletionToolChoiceObjectParam( + type="function", function=ChatCompletionToolChoiceFunctionParam(name=name) + ) + case _: + return None + + def _tool_choice_to_send( + self, model: str, tool_choice: object, drop_params: bool + ) -> str | ChatCompletionToolChoiceObjectParam | UnsupportedParamsError | None: + mapped: Final = self._map_tool_choice(tool_choice) + if mapped is not None: + return mapped + if drop_params or litellm.drop_params is True: + warn_dropped_reasoning( + "tool_choice", tool_choice, model, _UNUSABLE_TOOL_CHOICE_REASON, drop_params_gated=True + ) + return None + return UnsupportedParamsError( + message=( + f"Unsupported tool_choice={reprlib.repr(tool_choice)} for model={model}, since " + f"{_UNUSABLE_TOOL_CHOICE_REASON} " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) @staticmethod def _get_mistral_reasoning_system_prompt() -> str: @@ -244,18 +282,20 @@ class MistralConfig(OpenAIGPTConfig): optional_params["tools"] = self._clean_tool_schema_for_mistral(value) if param == "stream" and value is True: optional_params["stream"] = value - if param == "temperature": - optional_params["temperature"] = value - if param == "top_p": - optional_params["top_p"] = value - if param == "stop": - optional_params["stop"] = value - if param == "tool_choice" and isinstance(value, str): - optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value) + if param == "tool_choice" and "tools" in non_default_params and non_default_params["tools"]: + match self._tool_choice_to_send( + model, + cast(object, value), # cast-ok: untyped request params, the match narrows it + drop_params, + ): + case UnsupportedParamsError() as unusable_tool_choice: + raise unusable_tool_choice + case None: + pass + case tool_choice: + optional_params["tool_choice"] = tool_choice if param == "seed": optional_params["extra_body"] = {"random_seed": value} - if param == "response_format": - optional_params["response_format"] = value if ( param == "reasoning_effort" and "magistral" not in model.lower() @@ -271,7 +311,7 @@ class MistralConfig(OpenAIGPTConfig): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True _warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model) - if param in ("parallel_tool_calls", "prompt_cache_key"): + if param in ("temperature", "top_p", "stop", "response_format", "parallel_tool_calls", "prompt_cache_key"): optional_params[param] = value return optional_params diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index cafeece0a4e..d2ef33b89dd 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -13,24 +13,26 @@ The handler is intentionally a thin orchestration layer: 2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs from the structured dict produced by :class:`NvidiaRivaAudioTranscriptionConfig`. -3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS) - vs self-hosted (any host:port, optional TLS) modes. -4. Stream the audio through Riva's ``streaming_response_generator`` and - aggregate ``is_final`` results into a single transcript, or, when - ``riva_offline`` is set, send the whole buffer through the unary - ``offline_recognize`` RPC for NIMs that only ship offline engines. +3. Open a gRPC channel honoring NVCF (TLS by default) vs self-hosted (any + host:port, optional TLS) modes, and attach the ``authorization`` and NVCF + ``function-id`` metadata to each call exactly once. +4. Stream the audio through Riva's ``StreamingRecognize`` RPC and aggregate + ``is_final`` results into a single transcript, or, when ``riva_offline`` + is set, send the whole buffer through the unary ``Recognize`` RPC for NIMs + that only ship offline engines. 5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed on ``_hidden_params`` so cost calculation works. -``riva-client`` is imported lazily so ``litellm`` core remains usable -without the optional STT extras installed. +``grpcio`` and ``protobuf`` are imported lazily so ``litellm`` core remains +usable without the optional STT extras installed. """ +from __future__ import annotations + import asyncio -import inspect -from collections.abc import Callable, Iterable, Mapping, Sequence -from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol +from collections.abc import Iterable, Sequence +from contextlib import closing +from typing import TYPE_CHECKING, Final import httpx @@ -64,89 +66,19 @@ if TYPE_CHECKING: Logging as LiteLLMLoggingObj, ) -# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches -# NVIDIA's recommended chunk size for streaming ASR — small enough for -# responsive endpointing, large enough to keep per-RPC overhead low. -_DEFAULT_CHUNK_SAMPLES: Final = 1600 -_DEFAULT_CHUNK_BYTES: Final = _DEFAULT_CHUNK_SAMPLES * 2 # int16 = 2 bytes/sample - - _RIVA_INSTALL_HINT = "NVIDIA Riva client is not installed. Install with `pip install 'litellm[stt-nvidia-riva]'`." -class _RivaAuth(Protocol): - """Opaque ``riva.client.Auth`` handle.""" - - -class _RecognizeFuture(Protocol): - def result(self, timeout: float | None = None) -> object: ... - - def done(self) -> bool: ... - - def cancel(self) -> bool: ... - - -class _AsrService(Protocol): - @property - def streaming_response_generator(self) -> Callable[..., Iterable[object]]: ... - - def offline_recognize(self, audio_bytes: bytes, config: object, future: bool) -> _RecognizeFuture: ... - - -class _EndpointingConfig(Protocol): - """Opaque ``EndpointingConfig`` protobuf message.""" - - -class _EndpointingConfigField(Protocol): - CopyFrom: Callable[[_EndpointingConfig], None] - - -class _RecognitionConfig(Protocol): - @property - def endpointing_config(self) -> _EndpointingConfigField: ... - - -class _StreamingRecognitionConfig(Protocol): - """Opaque ``StreamingRecognitionConfig`` protobuf message.""" - - -class _AudioEncoding(Protocol): - @property - def LINEAR_PCM(self) -> object: ... - - -class _RivaClientModule(Protocol): - """The ``riva.client`` entry points this handler calls.""" - - @property - def Auth(self) -> Callable[..., _RivaAuth]: ... - - @property - def ASRService(self) -> Callable[[_RivaAuth], _AsrService]: ... - - -class _RivaAsrModule(Protocol): - """The protobuf constructors this handler calls, from whichever module exposes them.""" - - @property - def AudioEncoding(self) -> _AudioEncoding: ... - - @property - def RecognitionConfig(self) -> Callable[..., _RecognitionConfig]: ... - - @property - def StreamingRecognitionConfig(self) -> Callable[..., _StreamingRecognitionConfig]: ... - - @property - def EndpointingConfig(self) -> Callable[..., _EndpointingConfig]: ... - - -def _auth_factory(riva_module: _RivaClientModule) -> Callable[..., _RivaAuth]: - return riva_module.Auth - - -def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding: - return riva_asr_module.AudioEncoding +def _use_ssl(optional_params: dict) -> bool: + """ + NVCF endpoints are TLS-only, so ``nvcf_function_id`` defaults ``use_ssl`` + to True. Self-hosted Riva defaults to plaintext but honors an explicit + ``use_ssl``, since TLS-terminating ingresses in front of it are common. + """ + use_ssl_override: Final = optional_params.get("use_ssl") + if use_ssl_override is not None: + return bool(use_ssl_override) + return bool(optional_params.get("nvcf_function_id")) def _parse_riva_offline(value: object, drop_params: bool) -> bool: @@ -192,7 +124,7 @@ class NvidiaRivaAudioTranscription: litellm_params: dict, model_response: TranscriptionResponse, timeout: float | str | httpx.Timeout | None, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, api_key: str | None, api_base: str | None, atranscription: bool = False, @@ -237,13 +169,11 @@ class NvidiaRivaAudioTranscription: litellm_params: dict, model_response: TranscriptionResponse, timeout: float | str | httpx.Timeout | None, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, api_key: str | None, api_base: str | None, provider_config: NvidiaRivaAudioTranscriptionConfig | None = None, ) -> TranscriptionResponse: - # ``riva-client`` exposes a sync streaming generator, so we offload - # the blocking call to a worker thread to keep the event loop free. return await asyncio.to_thread( self._run_sync, model=model, @@ -267,7 +197,7 @@ class NvidiaRivaAudioTranscription: litellm_params: dict, model_response: TranscriptionResponse, timeout: float | str | httpx.Timeout | None, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, api_key: str | None, api_base: str | None, provider_config: NvidiaRivaAudioTranscriptionConfig, @@ -317,18 +247,12 @@ class NvidiaRivaAudioTranscription: ) or litellm.drop_params is True riva_offline: Final = _parse_riva_offline(optional_params.get("riva_offline"), drop_params=drop_params) - riva_module, riva_asr_module = _import_riva() - auth_obj: Final = self._construct_auth( - riva_module=riva_module, - api_base=api_base, - api_key=api_key, - optional_params=optional_params, - ) + try: + from litellm.llms.nvidia_riva.audio_transcription import riva_grpc + except ImportError as e: + raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e - recognition_config: Final = self._build_recognition_config_proto( - riva_asr_module=riva_asr_module, - recognition_config_dict=recognition_config_dict, - ) + recognition_config: Final = riva_grpc.recognition_config(recognition_config_dict) logging_obj.pre_call( input=None, api_key=api_key, @@ -344,26 +268,18 @@ class NvidiaRivaAudioTranscription: }, ) + metadata: Final = riva_grpc.call_metadata(api_key, optional_params.get("nvcf_function_id")) try: - asr_service: Final[_AsrService] = riva_module.ASRService(auth_obj) - responses: Final = ( - self._offline_responses( - asr_service=asr_service, - pcm_bytes=resampled.pcm_bytes, - recognition_config=recognition_config, - timeout=None if timeout is None else _deadline_seconds(timeout, drop_params), + deadline: Final = None if timeout is None else _deadline_seconds(timeout, drop_params) + with closing(riva_grpc.open_channel(api_base, _use_ssl(optional_params))) as channel: + responses: Final = ( + riva_grpc.recognize(channel, recognition_config, resampled.pcm_bytes, metadata, deadline) + if riva_offline + else riva_grpc.streaming_recognize( + channel, recognition_config, resampled.pcm_bytes, metadata, deadline + ) ) - if riva_offline - else self._streaming_responses( - asr_service=asr_service, - riva_asr_module=riva_asr_module, - pcm_bytes=resampled.pcm_bytes, - recognition_config=recognition_config, - timeout=timeout, - drop_params=drop_params, - ) - ) - final_results: Final = self._collect_final_results(responses, require_final=not riva_offline) + final_results: Final = self._collect_final_results(responses, require_final=not riva_offline) except NvidiaRivaException: raise except Exception as e: @@ -400,142 +316,6 @@ class NvidiaRivaAudioTranscription: return final_response - def _construct_auth( - self, - riva_module: _RivaClientModule, - api_base: str, - api_key: str | None, - optional_params: dict, - ) -> _RivaAuth: - """ - Build a ``riva.client.Auth`` object. - - - When ``nvcf_function_id`` is provided we attach the NVCF - ``function-id`` and bearer ``authorization`` metadata, and default - ``use_ssl`` to True (NVCF endpoints are TLS-only). - - Otherwise (self-hosted) we default ``use_ssl`` to False but still - honor an explicit override — self-hosted Riva behind an ingress - with TLS termination is a real deployment topology. - """ - nvcf_function_id: Final = optional_params.get("nvcf_function_id") - use_ssl_override: Final = optional_params.get("use_ssl") - use_ssl: Final = bool(use_ssl_override) if use_ssl_override is not None else bool(nvcf_function_id) - - metadata: Final[list[tuple[str, str]]] = [] - if nvcf_function_id: - metadata.append(("function-id", str(nvcf_function_id))) - if api_key: - metadata.append(("authorization", f"Bearer {api_key}")) - - try: - return _auth_factory(riva_module)(uri=api_base, use_ssl=use_ssl, metadata_args=metadata) - except TypeError: - # Older riva-client signatures used positional-only args. - return _auth_factory(riva_module)(None, use_ssl, api_base, metadata) - - def _build_recognition_config_proto( - self, riva_asr_module: _RivaAsrModule, recognition_config_dict: dict[str, Any] - ) -> _RecognitionConfig: - encoding_name: Final = (recognition_config_dict.get("encoding") or "LINEAR_PCM").upper() - encoding_enum: Final[object] = getattr( - _audio_encoding(riva_asr_module), - encoding_name, - _audio_encoding(riva_asr_module).LINEAR_PCM, - ) - - config: Final[_RecognitionConfig] = riva_asr_module.RecognitionConfig( - encoding=encoding_enum, - sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]), - language_code=recognition_config_dict["language_code"], - audio_channel_count=int(recognition_config_dict["audio_channel_count"]), - enable_automatic_punctuation=bool(recognition_config_dict.get("enable_automatic_punctuation", True)), - enable_word_time_offsets=bool(recognition_config_dict.get("enable_word_time_offsets", False)), - max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)), - model=recognition_config_dict.get("model", "") or "", - verbatim_transcripts=bool(recognition_config_dict.get("verbatim_transcripts", False)), - profanity_filter=bool(recognition_config_dict.get("profanity_filter", False)), - ) - - endpointing: Final = recognition_config_dict.get("endpointing_config") - if isinstance(endpointing, dict) and endpointing: - try: - ep: Final[_EndpointingConfig] = riva_asr_module.EndpointingConfig(**endpointing) - config.endpointing_config.CopyFrom(ep) - except Exception: - # If the user supplied an unknown EndpointingConfig field - # (older Riva server), fall back to Riva's defaults rather - # than failing the whole request. - pass - - return config - - def _streaming_responses( - self, - asr_service: _AsrService, - riva_asr_module: _RivaAsrModule, - pcm_bytes: bytes, - recognition_config: _RecognitionConfig, - timeout: float | str | httpx.Timeout | None, - drop_params: bool, - ) -> Iterable[object]: - streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig( - config=recognition_config, interim_results=False - ) - return asr_service.streaming_response_generator( - audio_chunks=self._iter_audio_chunks(pcm_bytes), - streaming_config=streaming_config, - **self._timeout_kwargs(asr_service.streaming_response_generator, timeout, drop_params), - ) - - def _offline_responses( - self, - asr_service: _AsrService, - pcm_bytes: bytes, - recognition_config: _RecognitionConfig, - timeout: float | None, - ) -> Iterable[object]: - pending: Final = asr_service.offline_recognize(audio_bytes=pcm_bytes, config=recognition_config, future=True) - try: - return (pending.result(timeout=timeout),) - except Exception as e: - if pending.done(): - raise - pending.cancel() - raise NvidiaRivaException( - status_code=408, - message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds", - ) from e - - @classmethod - def _timeout_kwargs( - cls, rpc: Callable[..., object], timeout: float | str | httpx.Timeout | None, drop_params: bool - ) -> Mapping[str, float]: - # Forward the deadline so the call cannot block forever if the - # server stalls. Older riva-client versions do not accept a - # ``timeout`` kwarg, so pass it only when supported. - if timeout is None or not cls._supports_timeout_kwarg(rpc): - return MappingProxyType({}) - return MappingProxyType({"timeout": _deadline_seconds(timeout, drop_params)}) - - @staticmethod - def _supports_timeout_kwarg(callable_obj: Callable[..., object]) -> bool: - try: - sig: Final = inspect.signature(callable_obj) - except (TypeError, ValueError): - return False - params: Final = sig.parameters - if "timeout" in params: - return True - return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) - - @staticmethod - def _iter_audio_chunks(pcm_bytes: bytes): - for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES): - chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES] - if not chunk: - continue - yield chunk - @staticmethod def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> Sequence[RivaFinalResult]: """ @@ -570,27 +350,3 @@ class NvidiaRivaAudioTranscription: } ) return final_results - - -def _import_riva() -> tuple[_RivaClientModule, _RivaAsrModule]: - """ - Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``. - - We try the SDK first (preferred) and fall back to importing the proto - module separately when the SDK packaging changes between versions. - """ - try: - import riva.client as riva_client - except ImportError as e: - raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e - - riva_asr_module = riva_client - if not hasattr(riva_asr_module, "RecognitionConfig"): - try: - from riva.client.proto import riva_asr_pb2 - - riva_asr_module = riva_asr_pb2 - except ImportError as e: - raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e - - return riva_client, riva_asr_module diff --git a/litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py b/litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py new file mode 100644 index 00000000000..13af5820f07 --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py @@ -0,0 +1,199 @@ +""" +The part of NVIDIA Riva's speech recognition gRPC schema that LiteLLM sends and reads. + +Names and numbers match ``riva/proto/riva_asr.proto`` and ``riva/proto/riva_audio.proto`` in +https://github.com/nvidia-riva/common. Fields LiteLLM never sets or reads are left out, and the +wire format skips them. The messages live in their own descriptor pool so they never collide +with nvidia-riva-client's copies when both are imported in one process. +""" + +from dataclasses import dataclass +from enum import Enum, IntEnum +from typing import Final + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from google.protobuf.message import Message + +_PACKAGE: Final = "nvidia.riva.asr" +_SERVICE: Final = f"/{_PACKAGE}.RivaSpeechRecognition" +RECOGNIZE_METHOD: Final = f"{_SERVICE}/Recognize" +STREAMING_RECOGNIZE_METHOD: Final = f"{_SERVICE}/StreamingRecognize" + + +class _Kind(Enum): + FLOAT = "TYPE_FLOAT" + INT32 = "TYPE_INT32" + BOOL = "TYPE_BOOL" + STRING = "TYPE_STRING" + MESSAGE = "TYPE_MESSAGE" + BYTES = "TYPE_BYTES" + ENUM = "TYPE_ENUM" + + +class AudioEncoding(IntEnum): + ENCODING_UNSPECIFIED = 0 + LINEAR_PCM = 1 + + +LINEAR_PCM: Final = AudioEncoding.LINEAR_PCM.value + + +@dataclass(frozen=True, slots=True) +class _Field: + name: str + number: int + kind: _Kind + type_name: str = "" + repeated: bool = False + optional: bool = False + oneof: str = "" + + +_MESSAGES: Final[tuple[tuple[str, tuple[_Field, ...]], ...]] = ( + ( + "RecognitionConfig", + ( + _Field("encoding", 1, _Kind.ENUM, "AudioEncoding"), + _Field("sample_rate_hertz", 2, _Kind.INT32), + _Field("language_code", 3, _Kind.STRING), + _Field("max_alternatives", 4, _Kind.INT32), + _Field("profanity_filter", 5, _Kind.BOOL), + _Field("audio_channel_count", 7, _Kind.INT32), + _Field("enable_word_time_offsets", 8, _Kind.BOOL), + _Field("enable_automatic_punctuation", 11, _Kind.BOOL), + _Field("model", 13, _Kind.STRING), + _Field("verbatim_transcripts", 14, _Kind.BOOL), + _Field("endpointing_config", 25, _Kind.MESSAGE, "EndpointingConfig", optional=True), + ), + ), + ( + "EndpointingConfig", + ( + _Field("start_history", 1, _Kind.INT32, optional=True), + _Field("start_threshold", 2, _Kind.FLOAT, optional=True), + _Field("stop_history", 3, _Kind.INT32, optional=True), + _Field("stop_threshold", 4, _Kind.FLOAT, optional=True), + _Field("stop_history_eou", 5, _Kind.INT32, optional=True), + _Field("stop_threshold_eou", 6, _Kind.FLOAT, optional=True), + ), + ), + ( + "StreamingRecognitionConfig", + ( + _Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"), + _Field("interim_results", 2, _Kind.BOOL), + ), + ), + ( + "StreamingRecognizeRequest", + ( + _Field("streaming_config", 1, _Kind.MESSAGE, "StreamingRecognitionConfig", oneof="streaming_request"), + _Field("audio_content", 2, _Kind.BYTES, oneof="streaming_request"), + ), + ), + ( + "WordInfo", + ( + _Field("start_time", 1, _Kind.INT32), + _Field("end_time", 2, _Kind.INT32), + _Field("word", 3, _Kind.STRING), + ), + ), + ( + "SpeechRecognitionAlternative", + ( + _Field("transcript", 1, _Kind.STRING), + _Field("words", 3, _Kind.MESSAGE, "WordInfo", repeated=True), + ), + ), + ( + "StreamingRecognitionResult", + ( + _Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True), + _Field("is_final", 2, _Kind.BOOL), + ), + ), + ( + "StreamingRecognizeResponse", + (_Field("results", 1, _Kind.MESSAGE, "StreamingRecognitionResult", repeated=True),), + ), + ( + "RecognizeRequest", + ( + _Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"), + _Field("audio", 2, _Kind.BYTES), + ), + ), + ( + "SpeechRecognitionResult", + (_Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True),), + ), + ( + "RecognizeResponse", + (_Field("results", 1, _Kind.MESSAGE, "SpeechRecognitionResult", repeated=True),), + ), +) + + +def _oneof_names(fields: tuple[_Field, ...]) -> tuple[str, ...]: + declared: Final = tuple(dict.fromkeys(field.oneof for field in fields if field.oneof)) + synthetic: Final = tuple(f"_{field.name}" for field in fields if field.optional) + return declared + synthetic + + +def _field_proto(field: _Field, oneofs: tuple[str, ...]) -> descriptor_pb2.FieldDescriptorProto: + oneof: Final = f"_{field.name}" if field.optional else field.oneof + return descriptor_pb2.FieldDescriptorProto( + name=field.name, + number=field.number, + type=descriptor_pb2.FieldDescriptorProto.Type.Value(field.kind.value), + label=( + descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED + if field.repeated + else descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL + ), + type_name=f".{_PACKAGE}.{field.type_name}" if field.type_name else None, + oneof_index=oneofs.index(oneof) if oneof else None, + proto3_optional=field.optional or None, + ) + + +def _message_proto(name: str, fields: tuple[_Field, ...]) -> descriptor_pb2.DescriptorProto: + oneofs: Final = _oneof_names(fields) + return descriptor_pb2.DescriptorProto( + name=name, + field=[_field_proto(field, oneofs) for field in fields], + oneof_decl=[descriptor_pb2.OneofDescriptorProto(name=oneof) for oneof in oneofs], + ) + + +_FILE: Final = descriptor_pb2.FileDescriptorProto( + name="litellm/nvidia_riva/riva_asr.proto", + package=_PACKAGE, + syntax="proto3", + enum_type=[ + descriptor_pb2.EnumDescriptorProto( + name="AudioEncoding", + value=[ + descriptor_pb2.EnumValueDescriptorProto(name=encoding.name, number=encoding.value) + for encoding in AudioEncoding + ], + ) + ], + message_type=[_message_proto(name, fields) for name, fields in _MESSAGES], +) +_POOL: Final = descriptor_pool.DescriptorPool() +_POOL.AddSerializedFile(_FILE.SerializeToString()) + + +def _message_class(name: str) -> type[Message]: + return message_factory.GetMessageClass(_POOL.FindMessageTypeByName(f"{_PACKAGE}.{name}")) + + +RecognitionConfig: Final = _message_class("RecognitionConfig") +EndpointingConfig: Final = _message_class("EndpointingConfig") +StreamingRecognitionConfig: Final = _message_class("StreamingRecognitionConfig") +StreamingRecognizeRequest: Final = _message_class("StreamingRecognizeRequest") +StreamingRecognizeResponse: Final = _message_class("StreamingRecognizeResponse") +RecognizeRequest: Final = _message_class("RecognizeRequest") +RecognizeResponse: Final = _message_class("RecognizeResponse") diff --git a/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py b/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py new file mode 100644 index 00000000000..8367853673f --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py @@ -0,0 +1,113 @@ +""" +gRPC calls to NVIDIA Riva's speech recognition service. + +The ``authorization`` and NVCF ``function-id`` metadata ride on each call, never on the channel, so every +request carries them exactly once over both plaintext and TLS. +""" + +from collections.abc import Iterator, Mapping, Sequence +from typing import Any, Final, TypeAlias + +import grpc +from google.protobuf.message import Message +from pydantic import TypeAdapter + +from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema as schema +from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException + +# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches +# NVIDIA's recommended chunk size for streaming ASR, small enough for +# responsive endpointing, large enough to keep per-RPC overhead low. +_CHUNK_SAMPLES: Final = 1600 +_CHUNK_BYTES: Final = _CHUNK_SAMPLES * 2 + +CallMetadata: TypeAlias = tuple[tuple[str, str], ...] + +_ENDPOINTING_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def call_metadata(api_key: str | None, nvcf_function_id: object) -> CallMetadata: + function_id: Final = (("function-id", str(nvcf_function_id)),) if nvcf_function_id else () + authorization: Final = (("authorization", f"Bearer {api_key}"),) if api_key else () + return function_id + authorization + + +def open_channel(api_base: str, use_ssl: bool) -> grpc.Channel: + if use_ssl: + return grpc.secure_channel(api_base, grpc.ssl_channel_credentials()) + return grpc.insecure_channel(api_base) + + +def recognition_config(config: Mapping[str, Any]) -> Message: + endpointing: Final = _endpointing_config(config.get("endpointing_config")) + return schema.RecognitionConfig( + encoding=schema.LINEAR_PCM, + sample_rate_hertz=int(config["sample_rate_hertz"]), + language_code=config["language_code"], + audio_channel_count=int(config["audio_channel_count"]), + enable_automatic_punctuation=bool(config.get("enable_automatic_punctuation", True)), + enable_word_time_offsets=bool(config.get("enable_word_time_offsets", False)), + max_alternatives=int(config.get("max_alternatives", 1)), + model=config.get("model", "") or "", + verbatim_transcripts=bool(config.get("verbatim_transcripts", False)), + profanity_filter=bool(config.get("profanity_filter", False)), + **({} if endpointing is None else {"endpointing_config": endpointing}), + ) + + +def _endpointing_config(endpointing: object) -> Message | None: + try: + fields: Final = _ENDPOINTING_FIELDS.validate_python(endpointing) + return schema.EndpointingConfig(**fields) if fields else None + except (ValueError, TypeError): + # A missing, unknown or mistyped field falls back to Riva's default + # endpointing rather than failing the whole request. + return None + + +def _streaming_requests(config: Message, pcm_bytes: bytes) -> Iterator[Message]: + yield schema.StreamingRecognizeRequest( + streaming_config=schema.StreamingRecognitionConfig(config=config, interim_results=False) + ) + for offset in range(0, len(pcm_bytes), _CHUNK_BYTES): + yield schema.StreamingRecognizeRequest(audio_content=pcm_bytes[offset : offset + _CHUNK_BYTES]) + + +def streaming_recognize( + channel: grpc.Channel, + config: Message, + pcm_bytes: bytes, + metadata: CallMetadata, + timeout: float | None, +) -> Iterator[Message]: + rpc: Final = channel.stream_stream( + schema.STREAMING_RECOGNIZE_METHOD, + request_serializer=schema.StreamingRecognizeRequest.SerializeToString, + response_deserializer=schema.StreamingRecognizeResponse.FromString, + ) + return rpc(_streaming_requests(config, pcm_bytes), metadata=metadata, timeout=timeout) + + +def recognize( + channel: grpc.Channel, + config: Message, + pcm_bytes: bytes, + metadata: CallMetadata, + timeout: float | None, +) -> Sequence[Message]: + rpc: Final = channel.unary_unary( + schema.RECOGNIZE_METHOD, + request_serializer=schema.RecognizeRequest.SerializeToString, + response_deserializer=schema.RecognizeResponse.FromString, + ) + pending: Final = rpc.future(schema.RecognizeRequest(config=config, audio=pcm_bytes), metadata=metadata) + try: + return (pending.result(timeout=timeout),) + except Exception as e: + if pending.done(): + raise + pending.cancel() + raise NvidiaRivaException( + status_code=408, + message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds", + ) from e diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 6bbf3598bd8..3cb798b7318 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -669,7 +669,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): prompt_cache_options: object = None, ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]: validated_input: Final = self._validate_input_param( - self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) + self._drop_bridge_minted_reasoning_items(input), drop_params=drops_params(litellm_params) ) reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options @@ -951,7 +951,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized def _validate_input_param( - self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False + self, input: str | ResponseInputParam, *, drop_params: bool = False ) -> str | ResponseInputParam: """ Ensure all input fields if pydantic are converted to dict @@ -965,9 +965,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): # if it's pydantic, convert to dict if isinstance(item, BaseModel): validated_input.append( - replayable_responses_item( - item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by - ) + replayable_responses_item(item.model_dump(exclude_none=True), drops_set_created_by=drop_params) ) elif isinstance(item, dict): validated_input.append( @@ -975,7 +973,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): self._handle_reasoning_item(cast(dict[str, object], item)) if item.get("type") == "reasoning" else item, - drops_set_created_by=drops_set_created_by, + drops_set_created_by=drop_params, ) ) else: diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index afd27b0f696..e5e3744dd80 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -89,7 +89,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): """Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}.""" input = self._ensure_message_type(input) if model.startswith("preset/"): - input = self._validate_input_param(input, drops_set_created_by=drops_params(litellm_params)) + input = self._validate_input_param(input, drop_params=drops_params(litellm_params)) data: Final[dict] = { "preset": model[len("preset/") :], "input": input, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 25c7c3593ed..76fd85d001b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2125,6 +2125,7 @@ "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2213,6 +2214,7 @@ "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2301,6 +2303,7 @@ "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2389,6 +2392,7 @@ "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2472,6 +2476,7 @@ "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2560,6 +2565,7 @@ "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2958,6 +2964,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3002,6 +3009,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3046,6 +3054,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3090,6 +3099,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3134,6 +3144,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3178,6 +3189,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -4063,6 +4075,7 @@ "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -4204,6 +4217,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -16507,6 +16521,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -17042,6 +17057,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -21910,6 +21926,7 @@ "prompt_cache_min_tokens": 512, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -22068,6 +22085,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -36308,6 +36326,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -36331,6 +36350,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -36350,6 +36370,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -36385,6 +36406,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -36420,6 +36442,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -39774,6 +39797,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 8e-06, + "output_cost_per_image_token_batches": 4e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -50486,6 +50510,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -50563,6 +50588,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -52968,6 +52994,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53006,6 +53033,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53255,6 +53283,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -59906,6 +59935,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -60629,6 +60659,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -64719,7 +64750,8 @@ { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", - "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number. On mapped Claude entries from Anthropic, Azure Foundry, Bedrock and Vertex it only fills the keys an entry leaves out, so the entry's own values always win.", "model_info": { "mode": "chat", "max_input_tokens": 200000, @@ -64762,6 +64794,14 @@ "thinking_always_on": true } }, + { + "name": "claude-thinking-on-by-default", + "pattern": "claude-[a-z]+-[5-9](?!\\d)", + "description": "Claude at major 5 or higher, in any id shape that contains claude--. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.", + "model_info": { + "thinking_on_by_default": true + } + }, { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", @@ -69285,6 +69325,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69360,6 +69401,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69616,6 +69658,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69681,6 +69724,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -75432,6 +75476,7 @@ "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -75548,6 +75593,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77052,6 +77098,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77104,6 +77151,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77148,6 +77196,7 @@ "thinking_always_on": false, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77195,6 +77244,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77231,6 +77281,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77287,6 +77338,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77333,6 +77385,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77373,6 +77426,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77414,6 +77468,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77460,6 +77515,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77511,6 +77567,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77557,6 +77614,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77617,6 +77675,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77668,6 +77727,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77718,6 +77778,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77762,6 +77823,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -79521,6 +79583,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79735,6 +79798,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79901,6 +79965,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -81886,6 +81951,7 @@ "prompt_cache_min_tokens": 512.0, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, "supports_function_calling": true, @@ -85880,5 +85946,235 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "gpt-image-2.6-sunburst": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "gpt-image-2.6-flare": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-west-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock_mantle/xai.grok-4.7": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_flex": 2.75e-07, + "cache_read_input_token_cost_priority": 9.625e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "output_cost_per_token_batches": 3.3e-06, + "output_cost_per_token_flex": 3.3e-06, + "output_cost_per_token_priority": 1.155e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "together_ai/ricursive-ai/Ripple-65B-Preview": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.together.ai/v1/models", + "supports_prompt_caching": true } } diff --git a/litellm/models/plugin_upload.py b/litellm/models/plugin_upload.py index 41fbf901a03..bd5e85c79cc 100644 --- a/litellm/models/plugin_upload.py +++ b/litellm/models/plugin_upload.py @@ -1,6 +1,23 @@ -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass -from typing import Protocol +from datetime import datetime +from enum import Enum +from types import MappingProxyType +from typing import Literal, Protocol + + +def has_agent_skill_metadata(manifest: Mapping[str, object]) -> bool: + return isinstance(manifest.get("agent_skill"), dict) + + +def uploaded_plugin_source(base_url: str, name: str, digest: str) -> Mapping[str, str]: + return MappingProxyType( + { + "source": "archive", + "url": f"{base_url.rstrip('/')}/claude-code/plugins/{name}/archive/{digest}.zip", + "sha256": digest, + } + ) @dataclass(frozen=True, slots=True) @@ -19,11 +36,130 @@ class PluginArchiveMetadata: enabled: bool source: Mapping[str, str] manifest_json: str | None + id: str | None = None + version: str | None = None + description: str | None = None + has_agent_skill_metadata: bool = False + updated_at: datetime | None = None + + +@dataclass(frozen=True, slots=True) +class PluginPublication: + name: str + version: str | None + description: str | None + manifest: Mapping[str, object] + created_by: str | None + content: bytes | None = None + + +@dataclass(frozen=True, slots=True) +class PluginPublicationBase: + id: str + updated_at: datetime | None + + +@dataclass(frozen=True, slots=True) +class PublishedPlugin: + id: str + name: str + version: str + enabled: bool + updated_at: datetime + + +@dataclass(frozen=True, slots=True) +class PluginPublicationError: + kind: Literal["invalid_version", "conflict", "not_found"] + message: str + + +@dataclass(frozen=True, slots=True) +class PluginVersion: + id: str + version: str | None + description: str | None + manifest: Mapping[str, object] + published_at: datetime | None + published_by: str | None + archive_available: bool + + +@dataclass(frozen=True, slots=True) +class PluginSnapshot: + id: str + name: str + version: str | None + description: str | None + manifest: Mapping[str, object] + enabled: bool + created_at: datetime | None + updated_at: datetime | None + created_by: str | None + + +@dataclass(frozen=True, slots=True) +class PluginDownloader: + authenticated: bool + user_id: str | None + user_email: str | None + key_alias: str | None + team_id: str | None + team_alias: str | None + + +class PluginDownloadResult(Enum): + RECORDED = "recorded" + SKILL_NOT_FOUND = "skill_not_found" + + +@dataclass(frozen=True, slots=True) +class PluginDownloadEvent: + id: str + downloaded_at: datetime + version: str | None + digest: str + authenticated: bool + user_id: str | None + user_email: str | None + key_alias: str | None + team_id: str | None + team_alias: str | None + + +@dataclass(frozen=True, slots=True) +class PluginDownloadsResponse: + total_downloads: int + anonymous_downloads: int + downloads: tuple[PluginDownloadEvent, ...] + next_cursor: str | None class PluginUploadStore(Protocol): async def create(self, upload: PluginUpload) -> str | None: ... + async def get_plugin(self, name: str) -> PluginSnapshot | None: ... + + async def publish( + self, publication: PluginPublication, expected: PluginPublicationBase | None = None + ) -> PublishedPlugin | PluginPublicationError: ... + + async def versions(self, metadata: PluginArchiveMetadata, limit: int, offset: int) -> tuple[PluginVersion, ...]: ... + + async def version(self, metadata: PluginArchiveMetadata, version_id: str) -> PluginVersion | None: ... + + async def version_content(self, metadata: PluginArchiveMetadata, version_id: str) -> bytes | None: ... + async def get_metadata(self, name: str) -> PluginArchiveMetadata | None: ... async def get_content(self, metadata: PluginArchiveMetadata) -> bytes | None: ... + + def public_metadata(self) -> AsyncIterator[PluginArchiveMetadata]: ... + + async def record_download( + self, metadata: PluginArchiveMetadata, digest: str, downloader: PluginDownloader + ) -> PluginDownloadResult: ... + + async def downloads(self, name: str, limit: int, cursor: int | None) -> PluginDownloadsResponse | None: ... + + async def download_counts(self, plugin_ids: tuple[str, ...]) -> Mapping[str, int]: ... diff --git a/litellm/models/skill_submission.py b/litellm/models/skill_submission.py new file mode 100644 index 00000000000..09d722fa69f --- /dev/null +++ b/litellm/models/skill_submission.py @@ -0,0 +1,110 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import Literal, Protocol, TypeAlias + +from litellm.models.plugin_upload import ( + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + PublishedPlugin, +) +from litellm.types.proxy.skill_scanning import SkillScan + +SubmissionStatus: TypeAlias = Literal["pending_review", "approved", "rejected"] +SubmissionOperation: TypeAlias = Literal["create", "update"] + + +@dataclass(frozen=True, slots=True) +class SkillProposal: + name: str + version: str | None + description: str | None + manifest: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class SkillSubmission: + id: str + plugin_name: str + operation: SubmissionOperation + status: SubmissionStatus + proposal: SkillProposal + has_archive: bool + submitted_by: str + created_at: datetime + base_plugin_id: str | None + base_updated_at: datetime | None + reviewed_by: str | None + reviewed_at: datetime | None + rejection_reason: str | None + + +@dataclass(frozen=True, slots=True) +class SubmissionPage: + submissions: tuple[SkillSubmission, ...] + has_more: bool + + +@dataclass(frozen=True, slots=True) +class SubmissionLimitExceeded: + message: str + + +@dataclass(frozen=True, slots=True) +class SubmissionMissing: + pass + + +@dataclass(frozen=True, slots=True) +class SubmissionConflict: + message: str + + +@dataclass(frozen=True, slots=True) +class SubmissionReviewed: + submission: SkillSubmission + plugin: PublishedPlugin | None + published_source: Mapping[str, str] | None = None + + +SubmissionReviewResult: TypeAlias = SubmissionReviewed | SubmissionMissing | SubmissionConflict | PluginPublicationError + + +@dataclass(frozen=True, slots=True) +class SkillScanInput: + scan: SkillScan + source: Mapping[str, str] + content: bytes | None + + +class SkillSubmissionStore(Protocol): + async def get_scan(self, submission_id: str) -> SkillScan | SubmissionMissing | None: ... + + async def start_scan( + self, submission_id: str, scan: SkillScan + ) -> SkillScanInput | SubmissionMissing | SubmissionConflict: ... + + async def finish_scan(self, submission_id: str, scan: SkillScan) -> None: ... + + async def submit( + self, upload: PluginPublication, submitted_by: str, base: PluginPublicationBase | None + ) -> SkillSubmission | SubmissionLimitExceeded: ... + + async def list( + self, submitted_by: str | None, status: SubmissionStatus | None, limit: int, offset: int + ) -> SubmissionPage: ... + + async def get(self, submission_id: str, submitted_by: str | None) -> SkillSubmission | None: ... + + async def get_archive(self, submission_id: str, submitted_by: str | None) -> bytes | None: ... + + async def review( + self, + submission_id: str, + reviewer: str, + approve: bool, + reason: str | None, + *, + publication_base_url: str | None = None, + ) -> SubmissionReviewResult: ... diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 2f613396413..01c22dc6725 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -48,6 +48,15 @@ def _mount_app(prefix: str, attr_name: str = "app") -> Callable[["FastAPI", obje return _register +def _include_skill_hub(app: "FastAPI", module: object) -> None: + from litellm.proxy.discovery_endpoints.skill_hub_submissions import router + from litellm.proxy.discovery_endpoints.skill_scanning import router as scan_router + + _include_router()(app, module) + app.include_router(router) + app.include_router(scan_router) + + @dataclass(frozen=True) class LazyFeature: name: str @@ -283,6 +292,12 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.decisions_endpoints.endpoints", path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone", "/v1/extraction", "/extraction"), ), + LazyFeature( + name="skill_hub", + module_path="litellm.proxy.discovery_endpoints.skill_hub_endpoints", + path_prefixes=("/skill-hub",), + register_fn=_include_skill_hub, + ), LazyFeature( name="claude_code_marketplace", module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index ab85b6f8842..7d9e230f8ef 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7541,6 +7541,145 @@ "title": "PluginAuthor", "type": "object" }, + "PluginDownloadEvent": { + "properties": { + "authenticated": { + "title": "Authenticated", + "type": "boolean" + }, + "digest": { + "title": "Digest", + "type": "string" + }, + "downloaded_at": { + "format": "date-time", + "title": "Downloaded At", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Alias" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "user_email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Email" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "id", + "downloaded_at", + "version", + "digest", + "authenticated", + "user_id", + "user_email", + "key_alias", + "team_id", + "team_alias" + ], + "title": "PluginDownloadEvent", + "type": "object" + }, + "PluginDownloadsResponse": { + "properties": { + "anonymous_downloads": { + "title": "Anonymous Downloads", + "type": "integer" + }, + "downloads": { + "items": { + "$ref": "#/components/schemas/PluginDownloadEvent" + }, + "title": "Downloads", + "type": "array" + }, + "next_cursor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Cursor" + }, + "total_downloads": { + "title": "Total Downloads", + "type": "integer" + } + }, + "required": [ + "total_downloads", + "anonymous_downloads", + "downloads", + "next_cursor" + ], + "title": "PluginDownloadsResponse", + "type": "object" + }, "PluginListItem": { "description": "Plugin item in list responses.", "properties": { @@ -7912,7 +8051,7 @@ } ], "default": "1.0.0", - "description": "Semantic version", + "description": "Semantic version, defaults to 1.0.0 on creation", "title": "Version" } }, @@ -7950,7 +8089,7 @@ "type": "object" }, "UpdatePluginRequest": { - "description": "Request body for replacing an existing plugin.\n\nThe plugin name is the resource identity and is supplied as the path\nparameter, so it cannot be changed here. This is a full replace: omitted\nfields reset to their defaults, so version is cleared rather than\ndefaulting to the create-time \"1.0.0\".", + "description": "Request body for replacing an existing plugin.\n\nThe plugin name is the resource identity and is supplied as the path\nparameter, so it cannot be changed here. This is a full replace: omitted\nfields reset to their defaults, except version which automatically increments\nthe published patch version when omitted or null.", "properties": { "author": { "anyOf": [ @@ -7999,6 +8138,19 @@ "description": "Skill domain (e.g., 'Productivity')", "title": "Domain" }, + "expected_updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Published updated_at read by the editor. A stale value returns 409", + "title": "Expected Updated At" + }, "homepage": { "anyOf": [ { @@ -8072,7 +8224,7 @@ "type": "null" } ], - "description": "Semantic version; cleared if omitted", + "description": "Explicit higher SemVer, or omit/null for an automatic patch bump", "title": "Version" } }, @@ -8203,7 +8355,7 @@ } ], "default": "1.0.0", - "description": "Semantic version", + "description": "Semantic version, defaults to 1.0.0 on creation", "title": "Version" } }, @@ -8535,7 +8687,7 @@ ] }, "put": { - "description": "Update an existing plugin in the LiteLLM marketplace.\n\nThe plugin is identified by its name in the path, which is the resource\nidentity and cannot be changed here. This is a full replace, not a merge:\nthe manifest is rebuilt from the request body, so any optional field left\nout is reset to its default (e.g. an omitted version is cleared, not kept).\nSend the full desired state.\n\nReturns 404 if no plugin with the given name exists; use\nPOST /claude-code/plugins to create a new plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: Name of the plugin to update (path parameter)\n - source: Plugin source reference (github, url, git-subdir, or archive format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n - installation_preference: 'available', 'auto_install', or 'required' (optional)\n\nReturns:\n Update status (action is always \"updated\") and plugin information.\n\nExample:\n ```bash\n curl -X PUT http://localhost:4000/claude-code/plugins/my-plugin \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"2.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", + "description": "Update an existing plugin in the LiteLLM marketplace.\n\nThe plugin is identified by its name in the path, which is the resource\nidentity and cannot be changed here. This is a full replace, not a merge:\nthe manifest is rebuilt from the request body, so any optional field left\nout is reset to its default, except version which automatically increments\nthe published patch version when omitted or null.\nSend the full desired state.\n\nReturns 404 if no plugin with the given name exists; use\nPOST /claude-code/plugins to create a new plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: Name of the plugin to update (path parameter)\n - source: Plugin source reference (github, url, git-subdir, or archive format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n - installation_preference: 'available', 'auto_install', or 'required' (optional)\n\nReturns:\n Update status (action is always \"updated\") and plugin information.\n\nExample:\n ```bash\n curl -X PUT http://localhost:4000/claude-code/plugins/my-plugin \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"2.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", "operationId": "update_plugin_claude_code_plugins__plugin_name__put", "parameters": [ { @@ -8690,6 +8842,87 @@ ] } }, + "/claude-code/plugins/{plugin_name}/downloads": { + "get": { + "description": "Admin-only ZIP download totals and authenticated downloader history.\n\nAnonymous downloads contribute only to totals. Counts include repeat requests and client prefetches,\nstarting when tracking is deployed. External sources and installation completion are not observed.", + "operationId": "get_plugin_downloads_claude_code_plugins__plugin_name__downloads_get", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + }, + { + "description": "Maximum number of download events", + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 25, + "description": "Maximum number of download events", + "maximum": 100, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "description": "Previous next_cursor", + "in": "query", + "name": "cursor", + "required": false, + "schema": { + "anyOf": [ + { + "pattern": "^[1-9][0-9]{0,18}$", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Previous next_cursor", + "title": "Cursor" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PluginDownloadsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Plugin Downloads", + "tags": [ + "claude_code_marketplace" + ] + } + }, "/claude-code/plugins/{plugin_name}/enable": { "post": { "description": "Enable a disabled plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: The name of the plugin to enable", @@ -58849,6 +59082,2622 @@ } } }, + "skill_hub": { + "components": { + "schemas": { + "Body_replace_skill_archive_skill_hub_skills__name__upload_put": { + "properties": { + "expected_updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expected Updated At" + }, + "file": { + "contentMediaType": "application/octet-stream", + "description": "Replacement Agent Skills ZIP with the same declared skill name", + "title": "File", + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "file" + ], + "title": "Body_replace_skill_archive_skill_hub_skills__name__upload_put", + "type": "object" + }, + "Body_upload_skill_skill_hub_skills_upload_post": { + "properties": { + "file": { + "contentMediaType": "application/octet-stream", + "description": "Agent Skills ZIP with SKILL.md and optional supporting files", + "title": "File", + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "file" + ], + "title": "Body_upload_skill_skill_hub_skills_upload_post", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ImportSkillRequest": { + "properties": { + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Repository-relative skill directory", + "title": "Path" + }, + "ref": { + "default": "HEAD", + "description": "Git commit, tag or branch to import", + "title": "Ref", + "type": "string" + }, + "skill": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill name to select when a repository contains several skills", + "title": "Skill" + }, + "url": { + "description": "Public skills.sh skill URL or GitHub repository URL", + "title": "Url", + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Semantic version, or omit/null to start at 1.0.0", + "title": "Version" + } + }, + "required": [ + "url" + ], + "title": "ImportSkillRequest", + "type": "object" + }, + "PluginDownloadEvent": { + "properties": { + "authenticated": { + "title": "Authenticated", + "type": "boolean" + }, + "digest": { + "title": "Digest", + "type": "string" + }, + "downloaded_at": { + "format": "date-time", + "title": "Downloaded At", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Alias" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "user_email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Email" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "id", + "downloaded_at", + "version", + "digest", + "authenticated", + "user_id", + "user_email", + "key_alias", + "team_id", + "team_alias" + ], + "title": "PluginDownloadEvent", + "type": "object" + }, + "PluginDownloadsResponse": { + "properties": { + "anonymous_downloads": { + "title": "Anonymous Downloads", + "type": "integer" + }, + "downloads": { + "items": { + "$ref": "#/components/schemas/PluginDownloadEvent" + }, + "title": "Downloads", + "type": "array" + }, + "next_cursor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Cursor" + }, + "total_downloads": { + "title": "Total Downloads", + "type": "integer" + } + }, + "required": [ + "total_downloads", + "anonymous_downloads", + "downloads", + "next_cursor" + ], + "title": "PluginDownloadsResponse", + "type": "object" + }, + "SkillHubAgentSkill": { + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "digest": { + "title": "Digest", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "origin": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Origin" + } + }, + "required": [ + "name", + "description", + "digest", + "origin" + ], + "title": "SkillHubAgentSkill", + "type": "object" + }, + "SkillHubList": { + "properties": { + "count": { + "title": "Count", + "type": "integer" + }, + "next_offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Next Offset" + }, + "skills": { + "items": { + "$ref": "#/components/schemas/SkillHubSkill" + }, + "title": "Skills", + "type": "array" + } + }, + "required": [ + "skills", + "count", + "next_offset" + ], + "title": "SkillHubList", + "type": "object" + }, + "SkillHubMetadata": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "maxLength": 1024, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "expected_updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Published updated_at read by the editor. A stale value returns 409", + "title": "Expected Updated At" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Explicit higher SemVer, or omit/null for an automatic patch bump", + "title": "Version" + } + }, + "title": "SkillHubMetadata", + "type": "object" + }, + "SkillHubPendingSkill": { + "properties": { + "action": { + "const": "submitted", + "default": "submitted", + "title": "Action", + "type": "string" + }, + "proposal": { + "$ref": "#/components/schemas/SkillHubProposal" + }, + "status": { + "const": "pending_review", + "default": "pending_review", + "title": "Status", + "type": "string" + }, + "submission_id": { + "title": "Submission Id", + "type": "string" + } + }, + "required": [ + "submission_id", + "proposal" + ], + "title": "SkillHubPendingSkill", + "type": "object" + }, + "SkillHubProposal": { + "properties": { + "agent_skill": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubAgentSkill" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "name": { + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "source": { + "additionalProperties": { + "type": "string" + }, + "title": "Source", + "type": "object" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "name", + "description", + "version", + "source" + ], + "title": "SkillHubProposal", + "type": "object" + }, + "SkillHubPublication": { + "additionalProperties": false, + "properties": {}, + "title": "SkillHubPublication", + "type": "object" + }, + "SkillHubRejection": { + "additionalProperties": false, + "properties": { + "reason": { + "anyOf": [ + { + "maxLength": 2000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + } + }, + "title": "SkillHubRejection", + "type": "object" + }, + "SkillHubReviewResponse": { + "properties": { + "action": { + "enum": [ + "approved", + "rejected" + ], + "title": "Action", + "type": "string" + }, + "skill": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSkill" + }, + { + "type": "null" + } + ] + }, + "status": { + "const": "success", + "default": "success", + "title": "Status", + "type": "string" + }, + "submission": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + }, + "required": [ + "action", + "submission", + "skill" + ], + "title": "SkillHubReviewResponse", + "type": "object" + }, + "SkillHubSkill": { + "properties": { + "content_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content Url" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "download_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Gateway ZIP downloads, included in proxy-admin catalog listings", + "title": "Download Count" + }, + "download_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Download Url" + }, + "id": { + "title": "Id", + "type": "string" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "name": { + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "source": { + "additionalProperties": { + "type": "string" + }, + "title": "Source", + "type": "object" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "visibility": { + "enum": [ + "public", + "private" + ], + "title": "Visibility", + "type": "string" + } + }, + "required": [ + "id", + "name", + "visibility", + "source" + ], + "title": "SkillHubSkill", + "type": "object" + }, + "SkillHubSubmission": { + "properties": { + "base_skill_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Base Skill Id" + }, + "base_updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Base Updated At" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "has_archive": { + "title": "Has Archive", + "type": "boolean" + }, + "id": { + "title": "Id", + "type": "string" + }, + "operation": { + "enum": [ + "create", + "update" + ], + "title": "Operation", + "type": "string" + }, + "proposal": { + "$ref": "#/components/schemas/SkillHubProposal" + }, + "rejection_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Rejection Reason" + }, + "reviewed_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reviewed At" + }, + "reviewed_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reviewed By" + }, + "skill_name": { + "title": "Skill Name", + "type": "string" + }, + "status": { + "enum": [ + "pending_review", + "approved", + "rejected" + ], + "title": "Status", + "type": "string" + }, + "submitted_by": { + "title": "Submitted By", + "type": "string" + } + }, + "required": [ + "id", + "skill_name", + "operation", + "status", + "proposal", + "has_archive", + "submitted_by", + "created_at", + "base_skill_id", + "base_updated_at", + "reviewed_by", + "reviewed_at", + "rejection_reason" + ], + "title": "SkillHubSubmission", + "type": "object" + }, + "SkillHubSubmissionDetail": { + "properties": { + "current_skill": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSkill" + }, + { + "type": "null" + } + ] + }, + "submission": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + }, + "required": [ + "submission", + "current_skill" + ], + "title": "SkillHubSubmissionDetail", + "type": "object" + }, + "SkillHubSubmissionList": { + "properties": { + "count": { + "title": "Count", + "type": "integer" + }, + "has_more": { + "title": "Has More", + "type": "boolean" + }, + "submissions": { + "items": { + "$ref": "#/components/schemas/SkillHubSubmission" + }, + "title": "Submissions", + "type": "array" + } + }, + "required": [ + "submissions", + "count", + "has_more" + ], + "title": "SkillHubSubmissionList", + "type": "object" + }, + "SkillHubVersion": { + "properties": { + "content_kind": { + "enum": [ + "stored_archive", + "source_reference" + ], + "title": "Content Kind", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "download_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Download Url" + }, + "id": { + "title": "Id", + "type": "string" + }, + "manifest": { + "additionalProperties": true, + "title": "Manifest", + "type": "object" + }, + "published_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published At" + }, + "published_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published By" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "id", + "version", + "description", + "manifest", + "published_at", + "published_by", + "content_kind", + "download_url" + ], + "title": "SkillHubVersion", + "type": "object" + }, + "SkillHubVersions": { + "properties": { + "next_offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Next Offset" + }, + "versions": { + "items": { + "$ref": "#/components/schemas/SkillHubVersion" + }, + "title": "Versions", + "type": "array" + } + }, + "required": [ + "versions", + "next_offset" + ], + "title": "SkillHubVersions", + "type": "object" + }, + "SkillScan": { + "properties": { + "archive_sha256": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Archive Sha256" + }, + "completed_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Completed At" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "expires_at": { + "format": "date-time", + "title": "Expires At", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "report": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillScanReport" + }, + { + "type": "null" + } + ] + }, + "requested_by": { + "title": "Requested By", + "type": "string" + }, + "started_at": { + "format": "date-time", + "title": "Started At", + "type": "string" + }, + "status": { + "enum": [ + "running", + "completed", + "failed" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "status", + "requested_by", + "started_at", + "expires_at" + ], + "title": "SkillScan", + "type": "object" + }, + "SkillScanFinding": { + "properties": { + "explanation": { + "anyOf": [ + { + "maxLength": 8000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Explanation" + }, + "file": { + "maxLength": 2048, + "title": "File", + "type": "string" + }, + "message": { + "maxLength": 8000, + "title": "Message", + "type": "string" + }, + "remediation": { + "anyOf": [ + { + "maxLength": 8000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Remediation" + }, + "rule_id": { + "maxLength": 200, + "title": "Rule Id", + "type": "string" + }, + "severity": { + "enum": [ + "LOW", + "MEDIUM", + "HIGH", + "CRITICAL" + ], + "title": "Severity", + "type": "string" + }, + "start_line": { + "anyOf": [ + { + "minimum": 1.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Start Line" + } + }, + "required": [ + "rule_id", + "severity", + "message", + "file" + ], + "title": "SkillScanFinding", + "type": "object" + }, + "SkillScanReport": { + "properties": { + "complete": { + "title": "Complete", + "type": "boolean" + }, + "findings": { + "items": { + "$ref": "#/components/schemas/SkillScanFinding" + }, + "maxItems": 500, + "title": "Findings", + "type": "array" + }, + "limitations": { + "default": [], + "items": { + "maxLength": 2000, + "type": "string" + }, + "maxItems": 100, + "title": "Limitations", + "type": "array" + }, + "risk_score": { + "maximum": 100.0, + "minimum": 0.0, + "title": "Risk Score", + "type": "integer" + }, + "scan_mode": { + "enum": [ + "static-only", + "static+llm" + ], + "title": "Scan Mode", + "type": "string" + }, + "scanner_version": { + "maxLength": 100, + "minLength": 1, + "title": "Scanner Version", + "type": "string" + } + }, + "required": [ + "scanner_version", + "complete", + "scan_mode", + "risk_score", + "findings" + ], + "title": "SkillScanReport", + "type": "object" + }, + "SkillScanRequest": { + "additionalProperties": false, + "properties": {}, + "title": "SkillScanRequest", + "type": "object" + }, + "SkillScanResponse": { + "properties": { + "configured": { + "title": "Configured", + "type": "boolean" + }, + "scan": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillScan" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "configured", + "scan" + ], + "title": "SkillScanResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "ctx": { + "title": "Context", + "type": "object" + }, + "input": { + "title": "Input" + }, + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/skill-hub/skills": { + "get": { + "description": "Browse the caller's public and granted private skills.", + "operationId": "list_skills_skill_hub_skills_get", + "parameters": [ + { + "in": "query", + "name": "query", + "required": false, + "schema": { + "default": "", + "title": "Query", + "type": "string" + } + }, + { + "in": "query", + "name": "offset", + "required": false, + "schema": { + "default": 0, + "minimum": 0, + "title": "Offset", + "type": "integer" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Skills", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/import": { + "post": { + "description": "Import a public skills.sh or GitHub skill as a portable snapshot.", + "operationId": "import_skill_skill_hub_skills_import_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportSkillRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSkill" + }, + { + "$ref": "#/components/schemas/SkillHubPendingSkill" + } + ], + "title": "Response Import Skill Skill Hub Skills Import Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Import Skill", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/upload": { + "post": { + "description": "Publish a portable skill while retaining Claude marketplace compatibility.", + "operationId": "upload_skill_skill_hub_skills_upload_post", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_upload_skill_skill_hub_skills_upload_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSkill" + }, + { + "$ref": "#/components/schemas/SkillHubPendingSkill" + } + ], + "title": "Response Upload Skill Skill Hub Skills Upload Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Upload Skill", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}": { + "delete": { + "description": "Delete a skill and its managed archive.", + "operationId": "remove_skill_skill_hub_skills__name__delete", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Remove Skill", + "tags": [ + "skill_hub" + ] + }, + "get": { + "description": "Read one visible skill and its portable download URL.", + "operationId": "get_skill_skill_hub_skills__name__get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSkill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Skill", + "tags": [ + "skill_hub" + ] + }, + "put": { + "description": "Replace registry metadata, preserving files. Members submit an immutable proposal for admin review.", + "operationId": "replace_skill_metadata_skill_hub_skills__name__put", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubMetadata" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSkill" + }, + { + "$ref": "#/components/schemas/SkillHubPendingSkill" + } + ], + "title": "Response Replace Skill Metadata Skill Hub Skills Name Put" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Replace Skill Metadata", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/archive": { + "get": { + "description": "Download the complete portable skill, with SKILL.md at the ZIP root.", + "operationId": "download_skill_skill_hub_skills__name__archive_get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + }, + { + "in": "query", + "name": "digest", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Digest" + } + } + ], + "responses": { + "200": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Download Skill", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/content": { + "get": { + "description": "Read the original SKILL.md without downloading supporting files.", + "operationId": "skill_content_skill_hub_skills__name__content_get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "text/markdown": {} + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Skill Content", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/publish": { + "post": { + "description": "Make a skill public using the existing Hub publication policy.", + "operationId": "publish_skill_skill_hub_skills__name__publish_post", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubPublication" + }, + { + "type": "null" + } + ], + "title": " Body" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSkill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Publish Skill", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/unpublish": { + "post": { + "description": "Remove a skill from public discovery while retaining explicit grants.", + "operationId": "unpublish_skill_skill_hub_skills__name__unpublish_post", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubPublication" + }, + { + "type": "null" + } + ], + "title": " Body" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSkill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Unpublish Skill", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/upload": { + "put": { + "description": "Replace skill files, retaining registry fields. Members propose an update against the original editor timestamp.", + "operationId": "replace_skill_archive_skill_hub_skills__name__upload_put", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_replace_skill_archive_skill_hub_skills__name__upload_put" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSkill" + }, + { + "$ref": "#/components/schemas/SkillHubPendingSkill" + } + ], + "title": "Response Replace Skill Archive Skill Hub Skills Name Upload Put" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Replace Skill Archive", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/versions": { + "get": { + "description": "Read published metadata history. External references are not retained artifacts.", + "operationId": "list_skill_versions_skill_hub_skills__name__versions_get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 20, + "maximum": 100, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "query", + "name": "offset", + "required": false, + "schema": { + "default": 0, + "minimum": 0, + "title": "Offset", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubVersions" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Skill Versions", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/versions/{version_id}": { + "get": { + "description": "Read a retained publication under the current skill's visibility policy.", + "operationId": "get_skill_version_skill_hub_skills__name__versions__version_id__get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + }, + { + "in": "path", + "name": "version_id", + "required": true, + "schema": { + "title": "Version Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubVersion" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Skill Version", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{name}/versions/{version_id}/archive": { + "get": { + "description": "Download a retained publication as a portable ZIP with SKILL.md at its root.", + "operationId": "download_skill_version_skill_hub_skills__name__versions__version_id__archive_get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + }, + { + "in": "path", + "name": "version_id", + "required": true, + "schema": { + "title": "Version Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Download Skill Version", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/skills/{plugin_name}/downloads": { + "get": { + "description": "Admin-only ZIP download totals and authenticated downloader history.\n\nAnonymous downloads contribute only to totals. Counts include repeat requests and client prefetches,\nstarting when tracking is deployed. External sources and installation completion are not observed.", + "operationId": "get_plugin_downloads_skill_hub_skills__plugin_name__downloads_get", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + }, + { + "description": "Maximum number of download events", + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 25, + "description": "Maximum number of download events", + "maximum": 100, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "description": "Previous next_cursor", + "in": "query", + "name": "cursor", + "required": false, + "schema": { + "anyOf": [ + { + "pattern": "^[1-9][0-9]{0,18}$", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Previous next_cursor", + "title": "Cursor" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PluginDownloadsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Plugin Downloads", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions": { + "get": { + "description": "Read a page of owned proposals. Proxy admins and admin viewers can read every owner's proposals.", + "operationId": "list_submissions_skill_hub_submissions_get", + "parameters": [ + { + "in": "query", + "name": "status", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "pending_review", + "approved", + "rejected" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 20, + "maximum": 100, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "query", + "name": "offset", + "required": false, + "schema": { + "default": 0, + "minimum": 0, + "title": "Offset", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmissionList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Submissions", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}": { + "get": { + "description": "Read an immutable proposal and its currently visible published skill.", + "operationId": "get_submission_skill_hub_submissions__submission_id__get", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmissionDetail" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Submission", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}/approve": { + "put": { + "description": "Publish a pending proposal and record its approval atomically. Only proxy admins may review.", + "operationId": "approve_submission_skill_hub_submissions__submission_id__approve_put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubReviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Approve Submission", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}/archive": { + "get": { + "description": "Download an owned proposal as a private portable ZIP. Legacy nonportable archives return 404.", + "operationId": "download_submission_skill_hub_submissions__submission_id__archive_get", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Download Submission", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}/reject": { + "put": { + "description": "Reject a pending proposal without changing the published skill. Only proxy admins may review.", + "operationId": "reject_submission_skill_hub_submissions__submission_id__reject_put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubRejection" + }, + { + "type": "null" + } + ], + "title": "Body" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubReviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Reject Submission", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}/scan": { + "get": { + "description": "Read the latest manual security scan. Only proxy admins may read reports.", + "operationId": "get_skill_scan_skill_hub_submissions__submission_id__scan_get", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillScanResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Skill Scan", + "tags": [ + "skill_hub" + ] + }, + "post": { + "description": "Start an optional scan of a pending submission. Scans never change approval decisions.", + "operationId": "start_skill_scan_skill_hub_submissions__submission_id__scan_post", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillScanRequest" + }, + { + "type": "null" + } + ], + "title": " Request" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillScanResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Start Skill Scan", + "tags": [ + "skill_hub" + ] + } + } + } + }, "tools": { "components": { "schemas": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6de663453d6..88094028df8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -89,6 +89,7 @@ from litellm.types.proxy.carried_budget_state import ( from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.ldap_sync import LDAPSyncSettings from litellm.types.proxy.request_tag_mappings import RequestTagMapping +from litellm.types.proxy.skill_scanning import SkillScannerConfig from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterConfig from litellm.types.router_weights import validate_router_settings_dict @@ -450,6 +451,28 @@ class LiteLLMRoutes(enum.Enum): "/v1/traces/{trace_id}/spans/{span_id}/error", ) + skill_hub_routes = ( + "/skill-hub/skills", + "/skill-hub/skills/upload", + "/skill-hub/skills/import", + "/skill-hub/skills/{name}", + "/skill-hub/skills/{name}/archive", + "/skill-hub/skills/{name}/content", + "/skill-hub/skills/{plugin_name}/downloads", + "/skill-hub/skills/{name}/versions", + "/skill-hub/skills/{name}/versions/{version_id}", + "/skill-hub/skills/{name}/versions/{version_id}/archive", + "/skill-hub/skills/{name}/publish", + "/skill-hub/skills/{name}/unpublish", + "/skill-hub/skills/{name}/upload", + "/skill-hub/submissions", + "/skill-hub/submissions/{submission_id}", + "/skill-hub/submissions/{submission_id}/archive", + "/skill-hub/submissions/{submission_id}/approve", + "/skill-hub/submissions/{submission_id}/reject", + "/skill-hub/submissions/{submission_id}/scan", + ) + litellm_native_routes = [ "/rag/ingest", "/v1/rag/ingest", @@ -469,6 +492,7 @@ class LiteLLMRoutes(enum.Enum): "/claude-code/plugins", "/claude-code/plugins/upload", "/claude-code/plugins/{plugin_name}/archive/{digest}.zip", + "/claude-code/plugins/{plugin_name}/downloads", "/claude-code/plugins/{plugin_name}", ] @@ -699,6 +723,7 @@ class LiteLLMRoutes(enum.Enum): + key_management_routes + mcp_management_routes + list(agent_management_routes) + + list(skill_hub_routes) ) spend_tracking_routes = [ @@ -941,6 +966,7 @@ class LiteLLMRoutes(enum.Enum): # Agent registry - reads are role-scoped and writes are proxy-admin-gated # inside agent_endpoints/endpoints.py *agent_management_routes, + *skill_hub_routes, ] # routes that manage their own allowed/disallowed logic ## Org Admin Routes ## @@ -2486,6 +2512,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="Periodically synchronize LDAP groups to teams and existing users' memberships. Requires Redis.", ) + use_spend_logs_ledger: bool = Field( + False, + description="Apply request accounting from stored SpendLogs deliveries. Enable only after all accounting writers are upgraded and ledger activation gates pass.", + ) + request_tag_mappings: Sequence[RequestTagMapping] | None = Field( None, description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.", @@ -3020,6 +3051,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, forwards client headers (e.g. Authorization) to the LLM API. Required for Claude Code with Max subscription.", ) + skill_scanner: SkillScannerConfig | None = Field( + None, description="Opt-in external scanner for manual admin review of skill submissions" + ) mcp_required_fields: list[str] | None = Field( None, description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).", @@ -3806,6 +3840,9 @@ class SpendLogsMetadata(TypedDict): class SpendLogsPayload(TypedDict): + accounting_payload: NotRequired[Mapping[str, object]] # writable-ok: capture before buffering + accounting_log_disabled: NotRequired[bool] # writable-ok: capture logging mode before buffering + accounting_delivery_id: NotRequired[str] # writable-ok: assigned once before buffering request_id: str call_type: str api_key: str diff --git a/litellm/proxy/alerting_endpoints/__init__.py b/litellm/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..6a7a204b435 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/__init__.py @@ -0,0 +1 @@ +"""HTTP endpoints for gateway alerts.""" diff --git a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py new file mode 100644 index 00000000000..a69dd6ee025 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py @@ -0,0 +1,109 @@ +"""Shared acknowledgement state for model migration alerts.""" + +import hashlib +import json +from datetime import date, datetime, timezone +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import TypeAdapter + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.table_repositories import DailyTagSpendRepository +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.model_deprecation import DeprecationAlertState, DeprecationAlertUpdate, DeprecationTagUsage + +router: Final = APIRouter() + + +def alert_state_repository(database: DatabaseDep) -> ConfigRepository: + if database is None: + raise HTTPException(status_code=503, detail="Alert state requires a database connection") + return ConfigRepository(database.prisma, use_writer=True) + + +def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository: + if database is None: + raise HTTPException(status_code=503, detail="Model usage tags require a database connection") + return DailyTagSpendRepository(database.prisma) + + +def _state_key(model: str, deprecation_date: date) -> str: + identity: Final = json.dumps([model, deprecation_date.isoformat()]) + digest: Final = hashlib.sha256(identity.encode()).hexdigest() + return f"model_deprecation_alert:{digest}" + + +def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None: + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if not write and user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + return + raise HTTPException(status_code=403, detail="Only gateway admins can manage model deprecation alerts") + + +@router.get("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState) +async def get_deprecation_alert_state( + model: Annotated[str, Query(min_length=1, max_length=1024)], + deprecation_date: date, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[ConfigRepository, Depends(alert_state_repository)], +) -> DeprecationAlertState: + _require_admin(user, write=False) + record: Final = await repository.get_param(_state_key(model, deprecation_date)) + if record is None: + return DeprecationAlertState(model=model, deprecation_date=deprecation_date) + return DeprecationAlertState.model_validate(record.param_value) + + +@router.put("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState) +async def update_deprecation_alert_state( + data: DeprecationAlertUpdate, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[ConfigRepository, Depends(alert_state_repository)], +) -> DeprecationAlertState: + _require_admin(user, write=True) + state: Final = DeprecationAlertState( + **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id + ) + await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")) + return state + + +@router.get("/alerting/model-deprecations/tags", tags=["alerting"], response_model=tuple[DeprecationTagUsage, ...]) +async def get_deprecation_usage_tags( + model: Annotated[str, Query(min_length=1, max_length=1024)], + api_keys: Annotated[tuple[str, ...], Query(min_length=1, max_length=100)], + start_date: date, + end_date: date, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[DailyTagSpendRepository, Depends(alert_tag_repository)], +) -> tuple[DeprecationTagUsage, ...]: + """Distinct tags observed with this model for a page of affected reporting key IDs.""" + _require_admin(user, write=False) + if not 0 <= (end_date - start_date).days < 30: + raise HTTPException(status_code=422, detail="Select up to 30 consecutive calendar dates") + rows: Final = await repository.table.group_by( + by=["api_key", "tag"], + where={ + "api_key": { + # bounded-ok: query validation limits api_keys to 100 IDs. + "in": [api_key_from_bucket(key) for key in api_keys] + }, + "model": model, + "date": {"gte": start_date.isoformat(), "lte": end_date.isoformat()}, + "api_requests": {"gt": 0}, + "tag": {"not": None}, + }, + ) + tags: Final = TypeAdapter(tuple[DeprecationTagUsage, ...]).validate_python(rows) + return tuple( + sorted( + (DeprecationTagUsage(api_key=api_key_bucket(usage.api_key), tag=usage.tag) for usage in tags), + key=lambda usage: (usage.api_key, usage.tag), + ) + ) diff --git a/litellm/proxy/anthropic_endpoints/answered_status.py b/litellm/proxy/anthropic_endpoints/answered_status.py index 8a3469f9119..26eeb08c8c9 100644 --- a/litellm/proxy/anthropic_endpoints/answered_status.py +++ b/litellm/proxy/anthropic_endpoints/answered_status.py @@ -1,10 +1,29 @@ from typing import Final, TypeVar -from litellm.exceptions import InternalServerError, MidStreamFallbackError +from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError + +_PROVIDER_OVERLOAD_STATUS_CODE: Final = 529 +_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE: Final = 503 _Raised = TypeVar("_Raised", bound=Exception) +def is_provider_overload(exc: Exception) -> bool: + return _provider_status_code(exc) == _PROVIDER_OVERLOAD_STATUS_CODE + + +def with_openai_overload_status(exc: _Raised) -> _Raised: + return ( + _copy_carrying( + _copy_carrying(exc, "provider_status_code", _PROVIDER_OVERLOAD_STATUS_CODE), + "status_code", + _OPENAI_OVERLOAD_ANSWERED_STATUS_CODE, + ) + if is_provider_overload(exc) + else exc + ) + + def _provider_status_code(exc: Exception) -> int | None: unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None: @@ -12,6 +31,10 @@ def _provider_status_code(exc: Exception) -> int | None: match unwrapped: case InternalServerError(provider_status_code=int() as provider_status): return provider_status + case APIError(provider_status_code=int() as provider_status): + return provider_status + case APIError(status_code=int() as provider_status): + return provider_status case _: return None diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index b8c90fe7e93..26cb0a2f3e6 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -27,14 +27,22 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import Annotated, Final, Protocol, TypedDict, get_args -from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import JSONResponse, Response from pydantic import Json, TypeAdapter -from typing_extensions import ReadOnly +from typing_extensions import ReadOnly, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.url_utils import is_https_url_with_host -from litellm.models.plugin_upload import PluginUpload +from litellm.models.plugin_upload import ( + PluginDownloader, + PluginDownloadResult, + PluginDownloadsResponse, + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + has_agent_skill_metadata, +) from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( SkillVisibility, @@ -125,7 +133,23 @@ async def get_prisma_client() -> object: _get_prisma_client: Final = get_prisma_client -async def _header_key_caller(request: Request) -> UserAPIKeyAuth: +def plugin_archive_url(request: Request, name: str, digest: str) -> str: + from litellm.proxy.utils import get_custom_url + + return get_custom_url( + request_base_url=str(request.base_url), route=f"claude-code/plugins/{name}/archive/{digest}.zip" + ) + + +def plugin_archive_source_matches( + request: Request, name: str, source: Mapping[str, str], digest: str, *, has_agent_skill_metadata: bool +) -> bool: + if has_agent_skill_metadata: + return source.get("source") == "archive" and source.get("sha256") == digest + return source == {"source": "archive", "url": plugin_archive_url(request, name, digest), "sha256": digest} + + +async def header_key_caller(request: Request) -> UserAPIKeyAuth: try: caller: Final = await user_api_key_auth( request=request, @@ -154,7 +178,7 @@ async def _header_key_caller(request: Request) -> UserAPIKeyAuth: async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAuth | None: if marketplace_auth_required(): - return await _header_key_caller(request) + return await header_key_caller(request) if key: return await user_api_key_auth(request=request, api_key=f"Bearer {key}") return None @@ -173,8 +197,6 @@ async def upload_plugin( metadata: Annotated[Json[UploadPluginRequest], Form(description="JSON skill metadata without a source URL")], ) -> RegisterPluginResponse: """Create a skill and its uploaded ZIP atomically. Duplicate names return 409.""" - from litellm.proxy.utils import get_custom_url - _require_proxy_admin(user_api_key_dict) if uploads is None: raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) @@ -189,16 +211,13 @@ async def upload_plugin( digest: Final = hashlib.sha256(archive).hexdigest() source: Final = { "source": "archive", - "url": get_custom_url( - request_base_url=str(request.base_url), - route=f"claude-code/plugins/{metadata.name}/archive/{digest}.zip", - ), + "url": plugin_archive_url(request, metadata.name, digest), "sha256": digest, } spec: Final = RegisterPluginRequest.model_validate({**metadata.model_dump(), "source": source}) _validate_installation_preference(spec) - plugin_id: Final = await uploads.create( - PluginUpload( + published: Final = await uploads.publish( + PluginPublication( name=spec.name, version=spec.version, description=spec.description, @@ -207,15 +226,15 @@ async def upload_plugin( created_by=user_api_key_dict.user_id, ) ) - if plugin_id is None: - raise _name_conflict_error(spec.name) + if isinstance(published, PluginPublicationError): + raise _publication_error(published) return RegisterPluginResponse( status="success", action="created", plugin=PluginResponse( - id=plugin_id, + id=published.id, name=spec.name, - version=spec.version, + version=published.version, description=spec.description, source=source, installation_preference=spec.installation_preference, @@ -224,6 +243,27 @@ async def upload_plugin( ) +def require_download_recorded(result: PluginDownloadResult) -> None: + match result: + case PluginDownloadResult.RECORDED: + return + case PluginDownloadResult.SKILL_NOT_FOUND: + raise _error_response(404, "Skill archive not found") + case _: + assert_never(result) + + +def plugin_downloader(caller: UserAPIKeyAuth | None) -> PluginDownloader: + return PluginDownloader( + authenticated=caller is not None, + user_id=caller.user_id if caller is not None else None, + user_email=caller.user_email if caller is not None else None, + key_alias=caller.key_alias if caller is not None else None, + team_id=caller.team_id if caller is not None else None, + team_alias=caller.team_alias if caller is not None else None, + ) + + @router.get( "/claude-code/plugins/{plugin_name}/archive/{digest}.zip", tags=["Claude Code Marketplace"], @@ -237,10 +277,8 @@ async def download_plugin_archive( uploads: PluginUploadRepositoryDep, ) -> Response: """Download an uploaded ZIP under the same visibility policy as its skill.""" - from litellm.proxy.utils import get_custom_url - caller: Final = ( - await _header_key_caller(request) + await header_key_caller(request) if marketplace_key_header_present(request) else await _marketplace_caller(request, None) ) @@ -248,18 +286,18 @@ async def download_plugin_archive( if uploads is None: raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) archive: Final = await uploads.get_metadata(plugin_name) - expected_url: Final = get_custom_url( - request_base_url=str(request.base_url), route=f"claude-code/plugins/{plugin_name}/archive/{digest}.zip" - ) if ( archive is None or not visibility.allows(archive) - or archive.source != {"source": "archive", "url": expected_url, "sha256": digest} + or not plugin_archive_source_matches( + request, plugin_name, archive.source, digest, has_agent_skill_metadata=archive.has_agent_skill_metadata + ) ): raise _error_response(404, "Skill archive not found") content: Final = await uploads.get_content(archive) if content is None: raise _error_response(404, "Skill archive not found") + require_download_recorded(await uploads.record_download(archive, digest, plugin_downloader(caller))) return Response( content=content, media_type="application/zip", @@ -271,6 +309,35 @@ async def download_plugin_archive( ) +@router.get( + "/claude-code/plugins/{plugin_name}/downloads", + tags=["Claude Code Marketplace"], + response_model=PluginDownloadsResponse, +) +async def get_plugin_downloads( + plugin_name: str, + uploads: PluginUploadRepositoryDep, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + limit: Annotated[int, Query(ge=1, le=100, description="Maximum number of download events")] = 25, + cursor: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,18}$", description="Previous next_cursor")] = None, +) -> PluginDownloadsResponse: + """Admin-only ZIP download totals and authenticated downloader history. + + Anonymous downloads contribute only to totals. Counts include repeat requests and client prefetches, + starting when tracking is deployed. External sources and installation completion are not observed. + """ + _require_proxy_admin(user_api_key_dict) + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) + before: Final = int(cursor) if cursor is not None else None + if before is not None and before > 9223372036854775807: + raise _error_response(422, "Invalid download cursor") + result: Final = await uploads.downloads(plugin_name, limit, before) + if result is None: + raise _error_response(404, "Plugin not found") + return result + + @router.get( "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], @@ -464,10 +531,14 @@ def _error_response(status_code: int, message: str) -> HTTPException: return HTTPException(status_code=status_code, detail={"error": message}) -def _name_conflict_error(name: str) -> HTTPException: - return _error_response( - 409, f"A skill named '{name}' already exists. Update the existing skill instead of adding it again." - ) +def _publication_error(error: PluginPublicationError) -> HTTPException: + match error.kind: + case "invalid_version": + return _error_response(400, error.message) + case "not_found": + return _error_response(404, error.message) + case "conflict": + return _error_response(409, error.message) def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: @@ -490,6 +561,7 @@ def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: ) async def register_plugin( request: RegisterPluginRequest, + uploads: PluginUploadRepositoryDep, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -532,12 +604,11 @@ async def register_plugin( }' ``` """ - from prisma.errors import UniqueViolationError - try: _require_proxy_admin(user_api_key_dict) - prisma_client: Final = await get_prisma_client() + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) if not re.match(r"^[a-z0-9-]+$", request.name): raise HTTPException( @@ -548,30 +619,19 @@ async def register_plugin( _validate_plugin_source(request.source) _validate_installation_preference(request) - existing: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( - where={"name": request.name} - ) - if existing: - raise _name_conflict_error(request.name) - manifest: Final[Mapping[str, object]] = _build_plugin_manifest(request.name, request) - try: - plugin: Final[_PluginRecord] = await ClaudeCodePluginRepository(prisma_client).table.create( - data={ - "name": request.name, - "version": request.version, - "description": request.description, - "manifest_json": json.dumps(manifest), - "files_json": "{}", - "enabled": True, - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - "created_by": user_api_key_dict.user_id, - } + plugin: Final = await uploads.publish( + PluginPublication( + name=request.name, + version=request.version, + description=request.description, + manifest=manifest, + created_by=user_api_key_dict.user_id, ) - except UniqueViolationError: - raise _name_conflict_error(request.name) + ) + if isinstance(plugin, PluginPublicationError): + raise _publication_error(plugin) verbose_proxy_logger.info("Plugin %s created successfully", request.name) @@ -582,7 +642,7 @@ async def register_plugin( id=plugin.id, name=plugin.name, version=plugin.version, - description=plugin.description, + description=request.description, source=request.source, installation_preference=request.installation_preference, enabled=plugin.enabled, @@ -608,7 +668,7 @@ async def register_plugin( async def list_plugins( enabled_only: bool = False, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): +) -> ListPluginsResponse: """ List all plugins in the marketplace. @@ -640,6 +700,7 @@ async def list_plugins( version=p.version, description=p.description, source=manifest.get("source", {}), + has_agent_skill_metadata=has_agent_skill_metadata(MANIFEST_FIELDS.validate_python(manifest)), author=manifest.get("author"), homepage=manifest.get("homepage"), keywords=manifest.get("keywords"), @@ -746,6 +807,7 @@ async def get_plugin( async def update_plugin( plugin_name: str, request: UpdatePluginRequest, + uploads: PluginUploadRepositoryDep, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): """ @@ -754,7 +816,8 @@ async def update_plugin( The plugin is identified by its name in the path, which is the resource identity and cannot be changed here. This is a full replace, not a merge: the manifest is rebuilt from the request body, so any optional field left - out is reset to its default (e.g. an omitted version is cleared, not kept). + out is reset to its default, except version which automatically increments + the published patch version when omitted or null. Send the full desired state. Returns 404 if no plugin with the given name exists; use @@ -792,6 +855,8 @@ async def update_plugin( try: _require_proxy_admin(user_api_key_dict) + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) prisma_client: Final = await get_prisma_client() @@ -799,26 +864,35 @@ async def update_plugin( existing: Final[_PluginRecord | None] = await repository.table.find_unique(where={"name": plugin_name}) if not existing: raise _error_response(404, f"Plugin '{plugin_name}' not found") + if request.expected_updated_at is not None and request.expected_updated_at != existing.updated_at: + raise _error_response(409, "The published skill changed. Reload it before publishing an update") current_manifest: Final = MANIFEST_FIELDS.validate_json(existing.manifest_json or "{}") if request.source != current_manifest.get("source"): _validate_plugin_source(request.source) _validate_installation_preference(request) - manifest: Final[Mapping[str, object]] = _build_plugin_manifest(plugin_name, request) + manifest: Final[Mapping[str, object]] = { + **_build_plugin_manifest(plugin_name, request), + **( + {"agent_skill": current_manifest["agent_skill"]} + if "agent_skill" in current_manifest and request.source == current_manifest.get("source") + else {} + ), + } - plugin: Final[_PluginRecord | None] = await repository.table.update( - where={"name": plugin_name}, - data={ - "version": request.version, - "description": request.description, - "manifest_json": json.dumps(manifest), - "files_json": "{}", - "updated_at": datetime.now(timezone.utc), - }, + plugin: Final = await uploads.publish( + PluginPublication( + name=plugin_name, + version=request.version, + description=request.description, + manifest=manifest, + created_by=user_api_key_dict.user_id, + ), + PluginPublicationBase(existing.id, existing.updated_at), ) - if plugin is None: - raise _error_response(404, f"Plugin '{plugin_name}' not found") + if isinstance(plugin, PluginPublicationError): + raise _publication_error(plugin) verbose_proxy_logger.info("Plugin %s updated successfully", plugin_name) @@ -829,7 +903,7 @@ async def update_plugin( id=plugin.id, name=plugin.name, version=plugin.version, - description=plugin.description, + description=request.description, source=request.source, installation_preference=request.installation_preference, enabled=plugin.enabled, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 17e6d3ee411..e812b03385d 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -95,6 +95,7 @@ from litellm.litellm_core_utils.streaming_handler import ( ) from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload, with_openai_overload_status from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, is_model_cost_zero, @@ -115,6 +116,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, + OverloadedProxyException, attribute_of, error_status_code, openai_error_param, @@ -164,6 +166,7 @@ _ChunkT: Final = TypeVar("_ChunkT") KNOWN_PROXY_ROUTES: Final = frozenset( route for member in LiteLLMRoutes for route in member.value if route.startswith("/") ) +_OPENAI_OVERLOAD_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion", "aresponses"}) ProxyRouteType: TypeAlias = Literal[ "acompletion", @@ -4398,28 +4401,28 @@ class ProxyBaseLLMRequestProcessing: version: str | None = None, ) -> Never: """Raises ProxyException (OpenAI API compatible) if an exception is raised""" - log_llm_api_exception(e, self.litellm_call_id) + answered: Final = with_openai_overload_status(e) if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES else e + log_llm_api_exception(answered, self.litellm_call_id) billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj") failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, - original_exception=e, + original_exception=answered, request_data=self.data, dispatched_call_type=self.route_type, ) # Use transformed exception if callback returned one, otherwise use original - if transformed_exception is not None: - e = transformed_exception - litellm_debug_info: Final = getattr(e, "litellm_debug_info", "") + failure: Final = transformed_exception if transformed_exception is not None else answered + litellm_debug_info: Final = getattr(failure, "litellm_debug_info", "") verbose_proxy_logger.debug( "An error occurred: %s %s", - e, + failure, litellm_debug_info, ) timeout: Final = getattr( - e, "timeout", None + failure, "timeout", None ) # returns the timeout set by the wrapper. Used for testing if model-specific timeout are set correctly _litellm_logging_obj: Final = ( billing_logging_obj if isinstance(billing_logging_obj, LiteLLMLoggingObj) else None @@ -4451,11 +4454,11 @@ class ProxyBaseLLMRequestProcessing: # a failed request reports no timing, matching /v1/chat/completions read_timing_from_logging_obj=False, ) - headers = getattr(e, "headers", None) or {} + headers = getattr(failure, "headers", None) or {} if not headers: - _response: Final = attribute_of(e, "response") + _response: Final = attribute_of(failure, "response") _response_headers: Final = getattr(_response, "headers", None) if _response is not None else None - _provider_headers: Final = _response_headers or getattr(e, "litellm_response_headers", None) + _provider_headers: Final = _response_headers or getattr(failure, "litellm_response_headers", None) if _provider_headers: headers = get_response_headers(dict(_provider_headers)) headers.update(custom_headers) @@ -4475,21 +4478,21 @@ class ProxyBaseLLMRequestProcessing: safe_headers: Final = {k: v for k, v in headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS} - self._apply_router_cooldown_retry_after(safe_headers, e) + self._apply_router_cooldown_retry_after(safe_headers, failure) - if isinstance(e, ProxyException): - e.headers = { - **{k: v for k, v in e.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}, + if isinstance(failure, ProxyException): + failure.headers = { + **{k: v for k, v in failure.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}, **{k: v if isinstance(v, str) else str(v) for k, v in safe_headers.items()}, } - raise e + raise failure - if isinstance(e, HTTPException): - raise proxy_exception_from_http_exception(e, safe_headers) - elif isinstance(e, httpx.HTTPStatusError): + if isinstance(failure, HTTPException): + raise proxy_exception_from_http_exception(failure, safe_headers) + elif isinstance(failure, httpx.HTTPStatusError): # Handle httpx.HTTPStatusError - extract actual error from response # This matches the original behavior before the refactor in commit 511d435f6f - http_status_error: Final[httpx.HTTPStatusError] = e + http_status_error: Final[httpx.HTTPStatusError] = failure error_body: Final = await http_status_error.response.aread() error_text: Final = error_body.decode("utf-8") @@ -4499,12 +4502,12 @@ class ProxyBaseLLMRequestProcessing: detail={"error": error_text}, headers=error_headers, ) - error_msg: Final = f"{e}" + error_msg: Final = f"{failure}" # Check for AttributeError in the exception chain. # The AttributeError may be wrapped in multiple layers # (e.g. AttributeError -> OpenAIException -> APIConnectionError), # so walk __cause__, __context__, and original_exception recursively. - has_attribute_error: Final = _has_attribute_error_in_chain(e) + has_attribute_error: Final = _has_attribute_error_in_chain(failure) if has_attribute_error: raise ProxyException( @@ -4519,12 +4522,12 @@ class ProxyBaseLLMRequestProcessing: # VertexAIError, etc.) all have a status_code attribute reflecting # the upstream API response. Use it to return the correct HTTP code # instead of defaulting to 500. - _exc_status_code: Final = getattr(e, "status_code", None) + _exc_status_code: Final = getattr(failure, "status_code", None) if _exc_status_code is not None and isinstance(_exc_status_code, int) and 400 <= _exc_status_code <= 599: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR - if should_report_bug(e): + if should_report_bug(failure): proxy_server_request: Final = self.data.get("proxy_server_request") request_url: Final = ( proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None @@ -4533,24 +4536,33 @@ class ProxyBaseLLMRequestProcessing: verbose_proxy_logger.error( bug_report_notice( build_proxy_bug_report( - e, + failure, call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), custom_llm_provider=self.data.get("custom_llm_provider"), stream=self.data.get("stream"), ) ) ) - client_message: Final = getattr(e, "message", error_msg) + client_message: Final = getattr(failure, "message", error_msg) + redacted_client_message: Final = redact_internal_details_from_client_message( + strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg + ) + provider_specific_fields: Final = getattr( + failure, "provider_specific_fields", None + ) or relayed_provider_specific_fields(failure) + if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES and is_provider_overload(failure): + raise OverloadedProxyException( + message=redacted_client_message, + headers=cast(dict[str, str], safe_headers), # cast-ok: ProxyException coerces header values to strings + provider_specific_fields=provider_specific_fields, + ) raise ProxyException( - message=redact_internal_details_from_client_message( - strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg - ), - type=openai_error_type(e, _code), - param=openai_error_param(e), - openai_code=getattr(e, "code", None), + message=redacted_client_message, + type=openai_error_type(failure, _code), + param=openai_error_param(failure), + openai_code=getattr(failure, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None) - or relayed_provider_specific_fields(e), + provider_specific_fields=provider_specific_fields, headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/ldap_directory.py b/litellm/proxy/common_utils/ldap_directory.py index 51c24a42949..e8aac4765b6 100644 --- a/litellm/proxy/common_utils/ldap_directory.py +++ b/litellm/proxy/common_utils/ldap_directory.py @@ -264,7 +264,9 @@ def read_directory( entries: Final = _entries(items) if isinstance(entries, DirectoryError): return entries - group_ids: Final = tuple(_one(entry, settings.group_id_attribute) for entry in entries) + group_ids: Final = tuple( + group_id for entry in entries if (group_id := _one(entry, settings.group_id_attribute)) is not None + ) if len(group_ids) != len(frozenset(group_ids)): return DirectoryError("LDAP group IDs are not unique within the selected groups") groups: Final = tuple(_read_group(client, settings, entry, cancellation) for entry in entries) diff --git a/litellm/proxy/common_utils/ldap_sync.py b/litellm/proxy/common_utils/ldap_sync.py index 83cc1c969ef..61085ecdc22 100644 --- a/litellm/proxy/common_utils/ldap_sync.py +++ b/litellm/proxy/common_utils/ldap_sync.py @@ -10,7 +10,7 @@ from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.common_utils.ldap_directory import DirectoryError, DirectoryGroup, read_directory +from litellm.proxy.common_utils.ldap_directory import DirectoryError, DirectoryGroup, LDAPConnection, read_directory from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.management_endpoints.team_endpoints import new_team @@ -89,8 +89,11 @@ class LDAPSync: prisma_client: PrismaClient, pod_lock_manager: PodLockManager, user_api_key_cache: UserApiKeyCache, - proxy_logging_obj: ProxyLogging, + proxy_logging_obj: ProxyLogging | None, + *, + directory_connection: LDAPConnection | None = None, ) -> None: + self.directory_connection = directory_connection self.settings = settings self.lease_seconds = math.ceil(settings.interval_minutes * 60) - 1 self.prisma_client = prisma_client @@ -157,7 +160,9 @@ class LDAPSync: update={"timeout_seconds": min(self.settings.timeout_seconds, max(1, self.lease_seconds // 4))} ) try: - snapshot: Final = await asyncio.to_thread(read_directory, directory_settings, stop=cancellation) + snapshot: Final = await asyncio.to_thread( + read_directory, directory_settings, connection=self.directory_connection, stop=cancellation + ) finally: cancellation.set() if isinstance(snapshot, DirectoryError): diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index b86ebd830f9..8a91c7b0a0c 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, timezone @@ -15,6 +16,7 @@ from litellm.types.proxy.model_deprecation import ( DeprecationStatus, ModelDeprecationInfo, ModelDeprecationResponse, + SuccessorModelSource, ) if TYPE_CHECKING: @@ -33,6 +35,7 @@ class _ResolvedDeprecation: litellm_provider: str | None copied_from_another_row: bool = False successor_model: str | None = None + deprecation_source: str | None = None def _parse_deprecation_date(raw_value: object) -> date | None: @@ -48,8 +51,8 @@ def _parse_deprecation_date(raw_value: object) -> date | None: return None -def _successor_model(raw_value: object) -> str | None: - """A non-empty successor name, or None when the field is missing, blank, or not a string""" +def _non_empty_string(raw_value: object) -> str | None: + """A non-empty trimmed string, or None for missing, blank, or non-string fields""" return raw_value.strip() or None if isinstance(raw_value, str) else None @@ -71,7 +74,8 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R copied_from_another_row=isinstance(entry, AliasedCostEntry) and "deprecation_date" not in entry.registered_fields and model_key not in catalog, - successor_model=_successor_model(fields.get("successor_model")), + successor_model=_non_empty_string(fields.get("successor_model")), + deprecation_source=_non_empty_string(fields.get("deprecation_source")), ) @@ -155,6 +159,7 @@ def _resolve_deployment_deprecation( litellm_model=model_key, litellm_provider=override_provider if isinstance(override_provider, str) else None, successor_model=from_catalog.successor_model if from_catalog is not None else None, + deprecation_source=from_catalog.deprecation_source if from_catalog is not None else None, ) @@ -166,14 +171,20 @@ def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus: return "upcoming" -def _checked_successor(model_name: str, model_info: Mapping[str, object], llm_router: Router) -> str | None: +def _checked_successor( + model_name: str, + model_info: Mapping[str, object], + llm_router: Router, + successor_log_level: int, +) -> str | None: """The deployment's successor_model as set, with a warning when no model in its team scope matches it""" - successor: Final = _successor_model(model_info.get("successor_model")) + successor: Final = _non_empty_string(model_info.get("successor_model")) if successor is None: return None team_id: Final = model_info.get("team_id") if not llm_router.registry().deployments(successor, team_id=team_id if isinstance(team_id, str) else None): - verbose_logger.warning( + verbose_logger.log( + successor_log_level, "model_deprecation: %s names successor %s, which matches no model group, alias, or wildcard route here", model_name, successor, @@ -187,6 +198,7 @@ def _build_info( today: date, warn_within_days: int, catalog: Mapping[str, object], + successor_log_level: int, ) -> _DeploymentInfo | None: model_name: Final = deployment.get("model_name") if not isinstance(model_name, str) or not model_name: @@ -197,16 +209,32 @@ def _build_info( return None days_until: Final = (resolved.deprecation_date - today).days - configured: Final = _checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router) + model_info: Final = _mapping_field(deployment, "model_info") + configured: Final = _checked_successor(model_name, model_info, llm_router, successor_log_level) + team_public_model_name_value: Final = model_info.get("team_public_model_name") + team_public_model_name: Final = ( + team_public_model_name_value + if isinstance(team_public_model_name_value, str) and team_public_model_name_value.strip() + else None + ) return _DeploymentInfo( info=ModelDeprecationInfo( model_name=model_name, + team_public_model_name=team_public_model_name, litellm_model=resolved.litellm_model, deprecation_date=resolved.deprecation_date, days_until_deprecation=days_until, status=_classify(days_until, warn_within_days), litellm_provider=resolved.litellm_provider, successor_model=configured or resolved.successor_model, + successor_model_source=( + SuccessorModelSource.DEPLOYMENT_CONFIG + if configured is not None + else SuccessorModelSource.MODEL_COST_MAP + if resolved.successor_model + else None + ), + deprecation_source=resolved.deprecation_source, ), configured_successor=configured is not None, ) @@ -218,13 +246,13 @@ class _DeploymentInfo: configured_successor: bool -def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo: +def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) -> ModelDeprecationInfo: """Keep the group's first deployment in config order, naming the first successor any of its deployments names, a successor configured in model_info ahead of one the cost map supplies""" first: Final = group[0].info tiers: Final = tuple( tuple( - deployment.info.successor_model + deployment.info for deployment in group if deployment.configured_successor is configured and deployment.info.successor_model is not None ) @@ -233,23 +261,33 @@ def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo: successors: Final = next((tier for tier in tiers if tier), ()) if not successors: return first - if len(frozenset(successors)) > 1: - verbose_logger.warning( + selected: Final = successors[0] + names: Final = frozenset(info.successor_model for info in successors if info.successor_model is not None) + if len(names) > 1: + verbose_logger.log( + successor_log_level, "model_deprecation: %s names several successors %s, reporting %s", first.model_name, - sorted(frozenset(successors)), - successors[0], + sorted(names), + selected.successor_model, ) - return first.model_copy(update={"successor_model": successors[0]}) + return first.model_copy( + update={ + "successor_model": selected.successor_model, + "successor_model_source": selected.successor_model_source, + "deprecation_source": selected.deprecation_source, + } + ) def _dedupe( models: Sequence[_DeploymentInfo], + successor_log_level: int, ) -> tuple[ModelDeprecationInfo, ...]: """Report a model group carrying the same date on several deployments once""" ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date)) return tuple( - _merge_group(tuple(group)) + _merge_group(tuple(group), successor_log_level) for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date)) ) @@ -264,11 +302,18 @@ def _bucket(models: Sequence[ModelDeprecationInfo], status: DeprecationStatus) - def _collect_from_router( - llm_router: Router, today: date, warn_within_days: int, catalog: Mapping[str, object] + llm_router: Router, + today: date, + warn_within_days: int, + catalog: Mapping[str, object], + successor_log_level: int, ) -> tuple[ModelDeprecationInfo, ...]: deployments: Final = llm_router.registry().deployments() or () - infos: Final = (_build_info(deployment, llm_router, today, warn_within_days, catalog) for deployment in deployments) - deduped: Final = _dedupe(tuple(info for info in infos if info is not None)) + infos: Final = ( + _build_info(deployment, llm_router, today, warn_within_days, catalog, successor_log_level) + for deployment in deployments + ) + deduped: Final = _dedupe(tuple(info for info in infos if info is not None), successor_log_level) verbose_logger.debug( "model_deprecation: %d/%d deployments carry a deprecation date", @@ -283,12 +328,13 @@ def collect_model_deprecations( warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, today: date | None = None, loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, + successor_log_level: int = logging.WARNING, ) -> ModelDeprecationResponse: """Bucket every deployment carrying a deprecation date by how urgent it is""" snapshot_time: Final = datetime.now(timezone.utc) effective_today: Final = today or snapshot_time.date() deduped: Final = ( - _collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog()) + _collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog(), successor_log_level) if llm_router is not None else () ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index a948c390375..26369bf3f87 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -17,6 +17,7 @@ from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id" +_SERVER_IS_OVERLOADED_CODE: Final = "server_is_overloaded" _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( { @@ -44,6 +45,31 @@ _FAILOVER_REFUSALS: Final = ( ) +class OverloadedProxyException(ProxyException): + def __init__( + self, + message: str, + headers: dict[str, str] | None, # mutable-ok: matches ProxyException's constructor + provider_specific_fields: dict[str, object] | None, # mutable-ok: matches ProxyException's constructor + ) -> None: + super().__init__( # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments + message=message, + type="service_unavailable_error", + param=None, + code=status.HTTP_503_SERVICE_UNAVAILABLE, + headers=None if headers is None else dict(headers), + provider_specific_fields=provider_specific_fields, + ) + + def to_dict(self) -> dict[str, object]: # mutable-ok: matches ProxyException.to_dict + return { + **_ERROR_BODY.validate_python( + super().to_dict() # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments + ), + "code": _SERVER_IS_OVERLOADED_CODE, + } + + def attribute_of(value: object, name: str, default: object = None) -> object: return getattr(value, name, default) diff --git a/litellm/proxy/config_resolvers/alerting.py b/litellm/proxy/config_resolvers/alerting.py index 4de7197f88b..1d7fbe0fffa 100644 --- a/litellm/proxy/config_resolvers/alerting.py +++ b/litellm/proxy/config_resolvers/alerting.py @@ -11,6 +11,7 @@ from typing import Final from litellm.proxy.config_resolvers._descriptors import FieldDescriptor EMAIL_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = ( + FieldDescriptor("PROXY_BASE_URL", "PROXY_BASE_URL", "PROXY_BASE_URL"), FieldDescriptor("SMTP_HOST", "SMTP_HOST", "SMTP_HOST"), FieldDescriptor("SMTP_PORT", "SMTP_PORT", "SMTP_PORT", default="587"), FieldDescriptor("SMTP_TLS", "SMTP_TLS", "SMTP_TLS", default="True"), diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py index 99b49331932..d2b69f24397 100644 --- a/litellm/proxy/db/budget_window_spend_writer.py +++ b/litellm/proxy/db/budget_window_spend_writer.py @@ -14,11 +14,13 @@ interval of increments queued elsewhere: the same lag the SpendLogs aggregate it replaces (and every other spend column) already has. """ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Final, Protocol +from pydantic import TypeAdapter from typing_extensions import LiteralString, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -74,30 +76,44 @@ _OPEN_WINDOW_SPEND_SQL: Final = ( "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')" ) +_HAS_DELIVERY: Final = 'EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id)' + +_SETTLED_SPEND_LOG_FILTER: Final = ( + ' AND NOT EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery ' + 'WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id ' + "AND delivery.state IN ('pending', 'parked'))" +) + _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, " - "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " + "COALESCE(SUM(spend) FILTER (WHERE " + + _HAS_DELIVERY + + " OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) " + "AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _SEED_FROM_SPEND_LOGS_TEAM_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, " - "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " + "COALESCE(SUM(spend) FILTER (WHERE " + + _HAS_DELIVERY + + " OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) " + "AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60) @@ -114,6 +130,10 @@ class _ExistingKeyRow(TypedDict): window_duration: ReadOnly[str] +_SEED_TOTALS_ROWS: Final = TypeAdapter(tuple[_SeedTotalsRow, ...]) +_EXISTING_KEY_ROWS: Final = TypeAdapter(tuple[_ExistingKeyRow, ...]) + + class _SeedTotalsQuery(Protocol): async def __call__(self, query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]: ... @@ -135,6 +155,8 @@ class _UpsertBatchManager(Protocol): class _UpsertTransaction(Protocol): def batch_(self) -> _UpsertBatchManager: ... + async def query_raw(self, query: LiteralString, *args: object) -> Sequence[Mapping[str, object]]: ... + class _UpsertTransactionManager(Protocol): async def __aenter__(self) -> _UpsertTransaction: ... @@ -178,6 +200,7 @@ class WindowSpendLogsAggregate(Protocol): entity_id: str, window_start: datetime, batch_started_at: datetime | None, + query: _SeedTotalsQuery | None = None, ) -> WindowSeedTotals | None: ... @@ -187,6 +210,7 @@ async def spend_logs_seed_totals( entity_id: str, window_start: datetime, batch_started_at: datetime | None, + query: _SeedTotalsQuery | None = None, ) -> WindowSeedTotals | None: """LiteLLM_SpendLogs spend for one entity since window_start, both in full and up to the start of the batch being flushed, in one scan. @@ -210,11 +234,12 @@ async def spend_logs_seed_totals( bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_TEAM_SQL, _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL else: return None + read: Final = query or _seed_totals_query(prisma_client) async with db_span("seed_window_spend", "LiteLLM_SpendLogs"): rows: Final = ( - await _seed_totals_query(prisma_client)(unbounded_sql, entity_id, window_start) + await read(unbounded_sql, entity_id, window_start) if batch_started_at is None - else await _seed_totals_query(prisma_client)( + else await read( bounded_sql, entity_id, window_start, @@ -246,9 +271,10 @@ def _primary_key(transaction: WindowSpendTransaction) -> tuple[str, str, str]: async def _existing_primary_keys( prisma_client: "PrismaClient", transactions: tuple[WindowSpendTransaction, ...], + query: _ExistingKeysQuery | None = None, ) -> frozenset[tuple[str, str, str]]: async with db_span("select_window_spend_rows", "LiteLLM_BudgetWindowSpend"): - rows: Final = await _existing_keys_query(prisma_client)( + rows: Final = await (query or _existing_keys_query(prisma_client))( _SELECT_EXISTING_ROWS_SQL, tuple(transaction["entity_type"] for transaction in transactions), tuple(transaction["entity_id"] for transaction in transactions), @@ -262,6 +288,7 @@ async def _seed_base_for_missing_row( transaction: WindowSpendTransaction, existing_primary_keys: frozenset[tuple[str, str, str]], spend_logs_aggregate: WindowSpendLogsAggregate, + query: _SeedTotalsQuery | None = None, ) -> float: """Spend already recorded for a window that has no row yet. @@ -272,12 +299,24 @@ async def _seed_base_for_missing_row( """ if _primary_key(transaction) in existing_primary_keys: return 0.0 - totals: Final = await spend_logs_aggregate( - prisma_client=prisma_client, - entity_type=transaction["entity_type"], - entity_id=transaction["entity_id"], - window_start=datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc), - batch_started_at=_transaction_started_at(transaction), + window_start: Final = datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc) + totals: Final = ( + await spend_logs_aggregate( + prisma_client=prisma_client, + entity_type=transaction["entity_type"], + entity_id=transaction["entity_id"], + window_start=window_start, + batch_started_at=_transaction_started_at(transaction), + ) + if query is None + else await spend_logs_aggregate( + prisma_client=prisma_client, + entity_type=transaction["entity_type"], + entity_id=transaction["entity_id"], + window_start=window_start, + batch_started_at=_transaction_started_at(transaction), + query=query, + ) ) if totals is None: return 0.0 @@ -324,10 +363,30 @@ def _upsert_params( ) +def _existing_keys_in(transaction: _UpsertTransaction) -> _ExistingKeysQuery: + async def query_raw(query: LiteralString, *args: object) -> Sequence[_ExistingKeyRow]: + return _EXISTING_KEY_ROWS.validate_python(await transaction.query_raw(query, *args)) + + return query_raw + + +def _seed_totals_in(transaction: _UpsertTransaction) -> _SeedTotalsQuery: + async def query_raw(query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]: + return _SEED_TOTALS_ROWS.validate_python(await transaction.query_raw(query, *args)) + + return query_raw + + +def _transaction_reads(transaction: _UpsertTransaction) -> tuple[_ExistingKeysQuery, _SeedTotalsQuery]: + """Reads made inside an enclosing transaction share its connection, so they cannot wait on it.""" + return _existing_keys_in(transaction), _seed_totals_in(transaction) + + async def commit_window_spend_updates( prisma_client: "PrismaClient", transactions: Sequence[WindowSpendTransaction], spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_seed_totals, + outer_transaction: _UpsertTransaction | None = None, ) -> None: """Apply aggregated window increments to LiteLLM_BudgetWindowSpend. @@ -343,9 +402,11 @@ async def commit_window_spend_updates( return ordered: Final = tuple(sorted(transactions, key=window_spend_group_key)) + reads: Final = _transaction_reads(outer_transaction) if outer_transaction is not None else (None, None) existing_primary_keys: Final = await _existing_primary_keys( prisma_client=prisma_client, transactions=ordered, + query=reads[0], ) seed_bases: Final = tuple( [ @@ -354,6 +415,7 @@ async def commit_window_spend_updates( transaction=transaction, existing_primary_keys=existing_primary_keys, spend_logs_aggregate=spend_logs_aggregate, + query=reads[1], ) for transaction in ordered ] @@ -367,7 +429,9 @@ async def commit_window_spend_updates( ) async with ( db_span("commit_window_spend_updates", "LiteLLM_BudgetWindowSpend"), - _upsert_transaction(prisma_client) as db_transaction, + ( + nullcontext(outer_transaction) if outer_transaction is not None else _upsert_transaction(prisma_client) + ) as db_transaction, db_transaction.batch_() as batcher, ): for transaction, seed_base in zip(ordered, seed_bases): diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index eb130a5196f..2d1770f4c1d 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -55,7 +55,7 @@ _ENTITY_INPUT_KEYS: Final[Mapping[DailySpendEntity, str]] = MappingProxyType( "tag": "request_tags", } ) -_TAGS: Final = TypeAdapter(tuple[str, ...]) +_TAGS: Final = TypeAdapter(tuple[object, ...]) def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEntity) -> tuple[str | None, ...]: @@ -67,7 +67,7 @@ def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEnti if value is None: return () tags: Final = _TAGS.validate_json(value) if isinstance(value, str) else _TAGS.validate_python(value) - return tuple(dict.fromkeys(tags)) + return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str))) if value is None: return (None,) if entity == "user" else () if not isinstance(value, str) or (entity == "end_user" and not value): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 35e7ae1c579..80f09df6c86 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,6 +12,7 @@ import json import os import random import time +import uuid from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar @@ -96,6 +97,7 @@ from litellm.types.utils import CallTypes from litellm.types.videos.utils import VIDEO_COST_REQUEST_ID_SUFFIX, VIDEO_STATUS_CALL_TYPES if TYPE_CHECKING: + from litellm.accounting.projection import Contribution from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution from litellm.proxy.utils import PrismaClient, ProxyLogging @@ -160,6 +162,18 @@ def _org_member_transaction_key(org_id: str, user_id: str) -> str: return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}" +ACCOUNTING_LEDGER_KWARG: Final = "accounting_ledger" + + +def _persistable_api_key(token: str | None, logged_api_key: str) -> str | None: + from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash + from litellm.proxy.spend_tracking.spend_tracking_utils import redact_logged_api_key + + if token is None or token == logged_api_key or is_valid_sha256_hash(token): + return token + return redact_logged_api_key(token) + + async def _track_guardrail_usage_of_cost_row( payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool ) -> None: @@ -200,12 +214,20 @@ _COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "star def _cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]: """Reduce a job's cost row to what tells its polls apart when logging is off. - A proxy run with spend logs disabled still needs one row per job to charge it once, + A legacy proxy run with spend logs disabled still needs one row per job to charge it once, so the row is written either way, but it carries no request of its own: no metadata, - no requester IP, no key, model, or token counts (LIT-7048). + no requester IP, no key, model, or token counts (LIT-7048). A ledger batch is deduplicated + by its delivery row instead, so it writes no local row when logging is off. """ if disable_spend_logs is False: return payload + if "accounting_payload" in payload: + return { + "request_id": payload["request_id"], + "call_type": payload["call_type"], + "accounting_payload": payload["accounting_payload"], + "accounting_log_disabled": True, + } return MappingProxyType({field: value for field, value in payload.items() if field in _COST_CLAIM_FIELDS}) @@ -219,8 +241,17 @@ class _MemberSpendRow(TypedDict): cost: ReadOnly[float] +class SpendUpsertTable(Protocol): + def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + class _SpendBatch(Protocol): litellm_usertable: BatchTable + litellm_endusertable: SpendUpsertTable + litellm_dailymodelusage: SpendUpsertTable + + def execute_raw(self, query: LiteralString, *args: object) -> None: ... + litellm_verificationtoken: BatchTable litellm_teamtable: BatchTable litellm_teammembership: BatchTable @@ -251,14 +282,14 @@ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) return _ENTITY_SPEND_TABLES[table_accessor](batcher) -class _SpendBatchManager(Protocol): +class SpendBatchManager(Protocol): async def __aenter__(self) -> _SpendBatch: ... async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... class _SpendTransaction(Protocol): - def batch_(self) -> _SpendBatchManager: ... + def batch_(self) -> SpendBatchManager: ... async def execute_raw(self, query: LiteralString, *args: object) -> int: ... @@ -413,8 +444,14 @@ _ENTITY_SPEND_MODELS: Final[Mapping[_EntitySpendTable, str]] = MappingProxyType( @asynccontextmanager async def _spend_update_tx( - prisma_client: PrismaClient, table: str, call_type: str = "commit_spend_updates" + prisma_client: PrismaClient, + table: str, + call_type: str = "commit_spend_updates", + outer_transaction: _SpendTransaction | None = None, ) -> AsyncGenerator[_SpendTransaction]: + if outer_transaction is not None: + yield outer_transaction + return tx: Final[_SpendTransactionManager] = prisma_client.db.tx(timeout=timedelta(seconds=60)) async with db_span(call_type, table), tx as transaction: await apply_rollup_lock_timeout(transaction) @@ -601,6 +638,21 @@ def debitable_model_access_groups( return tuple(group for group in ordered if group in served) +def _billable_cost(response_cost: float | None, payload: SpendLogsPayload) -> float | None: + billable_cost: Final = finite_cost(response_cost) + if response_cost is not None and billable_cost is None: + verbose_proxy_logger.warning( + "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", + response_cost, + single_log_line(payload["request_id"]), + ) + return billable_cost + + +def _iso_timestamp(value: datetime | str) -> str: + return value.isoformat() if isinstance(value, datetime) else value + + class DBSpendUpdateWriter: """ Module responsible for @@ -649,6 +701,7 @@ class DBSpendUpdateWriter: so the caller leaves the key, team, and user counters alone (LIT-7048). """ from litellm.proxy.proxy_server import ( + accounting_window_keys, disable_spend_logs, general_settings, # pyright: ignore[reportUnknownVariableType] # legacy proxy configuration is narrowed below litellm_proxy_budget_name, @@ -656,6 +709,13 @@ class DBSpendUpdateWriter: ) from litellm.proxy.utils import ProxyUpdateSpend, hash_token + proxy_settings: Final = cast( # cast-ok: proxy settings use string keys + Mapping[str, object], general_settings + ) + use_ledger: Final = proxy_settings.get("use_spend_logs_ledger") is True + request_fields: Final = _SPEND_METADATA_ADAPTER.validate_python(kwargs or {}) + if kwargs is not None: + kwargs[ACCOUNTING_LEDGER_KWARG] = use_ledger # rebind-ok: spend counters leave ledger windows to the worker try: verbose_proxy_logger.debug( "Enters prisma db call, response_cost: %s, token: %s; user_id: %s; team_id: %s", @@ -666,10 +726,9 @@ class DBSpendUpdateWriter: ) if ProxyUpdateSpend.disable_spend_updates() is True: return True - if token is not None and isinstance(token, str) and token.startswith("sk-"): - hashed_token = hash_token(token=token) - else: - hashed_token = token + hashed_token: Final = ( + hash_token(token=token) if isinstance(token, str) and token.startswith("sk-") else token + ) ## CREATE SPEND LOG PAYLOAD ## from litellm.proxy.spend_tracking.spend_tracking_utils import ( @@ -685,32 +744,42 @@ class DBSpendUpdateWriter: end_time=end_time, llm_router=llm_router, ) - billable_cost: Final = finite_cost(response_cost) - if response_cost is not None and billable_cost is None: - verbose_proxy_logger.warning( - "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", - response_cost, - single_log_line(payload["request_id"]), - ) + billable_cost: Final = _billable_cost(response_cost, payload) payload["spend"] = billable_cost or 0.0 - if isinstance(payload["startTime"], datetime): - payload["startTime"] = payload["startTime"].isoformat() - if isinstance(payload["endTime"], datetime): - payload["endTime"] = payload["endTime"].isoformat() - - if org_id is not None and org_id != "": + payload["startTime"] = _iso_timestamp(payload["startTime"]) + payload["endTime"] = _iso_timestamp(payload["endTime"]) + if org_id: payload["organization_id"] = org_id - - if team_id is not None and team_id != "": + if team_id: payload["team_id"] = team_id - error_log_settings: Final = cast( # cast-ok: proxy settings use string keys - Mapping[str, object], general_settings - ) + if use_ledger: + from litellm.accounting.snapshot import build_accounting_payload + + if not _is_settle_once_cost_row(payload): + payload["accounting_delivery_id"] = uuid.uuid4().hex + payload["accounting_payload"] = build_accounting_payload( + payload, + entities={ + "api_key": _persistable_api_key(hashed_token, payload.get("api_key") or ""), + "user": user_id, + "end_user": end_user_id, + "team_id": team_id, + "organization_id": org_id, + "project_id": project_id, + "agent_id": payload["agent_id"], + "proxy_budget_name": litellm_proxy_budget_name if litellm.max_budget > 0 else None, + }, + model_access_groups=debitable_model_access_groups( + get_request_model_access_groups(request_fields), payload["model_id"], llm_router + ), + windows=await accounting_window_keys(hashed_token, team_id) if billable_cost else (), + router=get_llm_router, + ) keep_spend_logs: Final = disable_spend_logs is False and not ( "status" in payload and payload["status"] == "failure" - and error_log_settings.get("disable_error_logs") is True + and proxy_settings.get("disable_error_logs") is True ) if not await self._record_spend_log( payload=payload, @@ -739,24 +808,26 @@ class DBSpendUpdateWriter: "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur." ) - spawn_logging_task( - asyncio.get_running_loop(), - self._batch_database_updates( - response_cost=billable_cost, - user_id=user_id, - hashed_token=hashed_token, - team_id=team_id, - org_id=org_id, - project_id=project_id, - end_user_id=end_user_id, - prisma_client=prisma_client, - litellm_proxy_budget_name=litellm_proxy_budget_name, - payload=payload, - request_model_access_groups=get_request_model_access_groups(kwargs), - ), - task_name="spend rollups", - ) - + if not use_ledger: + spawn_logging_task( + asyncio.get_running_loop(), + self._batch_database_updates( + response_cost=billable_cost, + user_id=user_id, + hashed_token=hashed_token, + team_id=team_id, + org_id=org_id, + project_id=project_id, + end_user_id=end_user_id, + prisma_client=prisma_client, + litellm_proxy_budget_name=litellm_proxy_budget_name, + payload=payload, + request_model_access_groups=get_request_model_access_groups(request_fields), + ), + task_name="spend rollups", + ) + else: + self._record_request_error_safely(payload=payload, prisma_client=prisma_client) self._enqueue_tool_registry_upsert( kwargs=kwargs, completion_response=completion_response, @@ -791,7 +862,13 @@ class DBSpendUpdateWriter: return await self._claim_settle_once_cost_row( payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs ) - if disable_spend_logs is False: + if "accounting_payload" in payload: + await self._insert_spend_log_to_db( + payload={**payload, "accounting_log_disabled": disable_spend_logs}, + prisma_client=prisma_client, + llm_router=llm_router, + ) + elif disable_spend_logs is False: await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router) return True @@ -804,7 +881,9 @@ class DBSpendUpdateWriter: the charge and every later one finds the row and charges nothing (LIT-7048). Only a row that recorded a charge counts: a failed poll, a request whose client picked the job id as its call id, and the $0 row a poll left behind - while the job was still running all leave the charge to be made. + while the job was still running all leave the charge to be made. A ledger job + is owned by its delivery row, keyed by its cost row id, in every logging mode, and a + $0 poll records no delivery so it cannot block the poll that carries the cost. """ from litellm.repositories.table_repositories import SpendLogsRepository @@ -817,13 +896,24 @@ class DBSpendUpdateWriter: row: Final = spend_log_row_with_retained_metadata( _cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields() ) - spend_logs: Final = SpendLogsRepository(prisma_client).table + repository: Final = SpendLogsRepository(prisma_client) try: - claimed: Final = await spend_logs.create_many( + if "accounting_payload" in payload: + if payload["spend"] <= 0: + return True + accepted: Final = await repository.insert_with_deliveries([prisma_client.jsonify_object(row)]) == 1 + if accepted: + await _track_guardrail_usage_of_cost_row(payload, prisma_client, disable_spend_logs) + return accepted + if await repository.has_delivery(request_id): + return False + claimed: Final = await repository.table.create_many( data=[prisma_client.jsonify_object(row)], skip_duplicates=True, ) - existing: Final = None if claimed == 1 else await spend_logs.find_unique(where={"request_id": request_id}) + existing: Final = ( + None if claimed == 1 else await repository.table.find_unique(where={"request_id": request_id}) + ) except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; an unreachable DB queues the row like any other spend log verbose_proxy_logger.warning( "Could not claim spend row %s for a job's cost, queueing it: %s", request_id, e @@ -1268,10 +1358,7 @@ class DBSpendUpdateWriter: self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client), payload=payload_copy, ) - try: - self._record_request_error(payload=payload_copy, prisma_client=prisma_client) - except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes - _report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e) + self._record_request_error_safely(payload=payload_copy, prisma_client=prisma_client) await self._run_spend_update_step( "add_spend_log_transaction_to_daily_org_transaction", self.add_spend_log_transaction_to_daily_org_transaction( @@ -1601,7 +1688,7 @@ class DBSpendUpdateWriter: async def _insert_spend_log_to_db( self, - payload: dict | SpendLogsPayload, + payload: Mapping[str, object], prisma_client: PrismaClient | None = None, spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"), llm_router: "Router | None" = None, @@ -1645,6 +1732,13 @@ class DBSpendUpdateWriter: else: - Regular flow of this method """ + from litellm.accounting.worker import flush_accounting_deliveries + + try: + await flush_accounting_deliveries(self, prisma_client, proxy_logging_obj) + except Exception: # noqa: BLE001 # pending deliveries survive outages while legacy queues must still drain + verbose_proxy_logger.exception("Accounting delivery flush failed; pending work will retry") + if RedisUpdateBuffer.should_commit_spend_updates_to_redis(): await self._commit_spend_updates_to_db_with_redis( prisma_client=prisma_client, @@ -2171,6 +2265,39 @@ class DBSpendUpdateWriter: _handle_spend_update_failure = handle_spend_update_failure + @staticmethod + async def invalidate_accounting_entities( + contributions: tuple["Contribution", ...], proxy_logging_obj: ProxyLogging + ) -> None: + await DBSpendUpdateWriter._invalidate_project_caches( + tuple(dict(item.key)["project_id"] for item in contributions if item.table == "LiteLLM_ProjectTable"), + proxy_logging_obj, + ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + details: Final = _SPEND_METADATA_ADAPTER.validate_python( + proxy_logging_obj.call_details # pyright: ignore[reportUnknownMemberType] # legacy logging metadata is validated at this boundary + ) + user_api_key_cache: Final = details.get("user_api_key_cache") + if isinstance(user_api_key_cache, UserApiKeyCache): + with service_target(AUTH_OBJECTS_TARGET): + for item in contributions: + if item.table == "LiteLLM_TeamMembership": + await user_api_key_cache.async_delete_cache( + key=f"team_membership:{dict(item.key)['user_id']}:{dict(item.key)['team_id']}" + ) + + async def apply_accounting_entities( + self, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + transaction: _SpendTransaction, + updates: DBSpendUpdateTransactions, + ) -> None: + await self._commit_spend_updates_to_db( + prisma_client, 0, proxy_logging_obj, updates, outer_transaction=transaction + ) + async def _commit_spend_updates_to_db( self, prisma_client: PrismaClient, @@ -2178,6 +2305,7 @@ class DBSpendUpdateWriter: proxy_logging_obj: ProxyLogging, db_spend_update_transactions: DBSpendUpdateTransactions, on_table_committed: Callable[[_SpendTableName], None] | None = None, + outer_transaction: _SpendTransaction | None = None, ): """ Commits all the spend `UPDATE` transactions to the Database @@ -2192,7 +2320,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_UserTable") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_UserTable", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by ID for consistent lock ordering across pods to prevent deadlocks. # batch_() issues statements sequentially within the tx, so iteration @@ -2204,6 +2334,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2223,6 +2355,7 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, end_user_list_transactions=end_user_list_transactions, + outer_transaction=outer_transaction, ) if on_table_committed is not None: on_table_committed("end_user_list_transactions") @@ -2233,7 +2366,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_VerificationToken") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_VerificationToken", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by token for consistent lock ordering across pods to prevent deadlocks. for token, response_cost in sorted(key_list_transactions.items()): @@ -2248,6 +2383,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2265,7 +2402,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_TeamTable") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_TeamTable", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by team_id for consistent lock ordering across pods to prevent deadlocks. for team_id, response_cost in sorted(team_list_transactions.items()): @@ -2278,6 +2417,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2303,11 +2444,15 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_TeamMembership") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_TeamMembership", outer_transaction=outer_transaction + ) as transaction: await _write_team_member_spend(transaction, team_member_list_transactions) # Transaction succeeded, break out of retry loop break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2320,7 +2465,7 @@ class DBSpendUpdateWriter: # Invalidate cache for updated team memberships # This ensures budget checks read fresh spend data from the database - if team_memberships_to_invalidate and proxy_logging_obj is not None: + if outer_transaction is None and team_memberships_to_invalidate and proxy_logging_obj is not None: user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache") if user_api_key_cache is not None: with service_target(AUTH_OBJECTS_TARGET): @@ -2340,7 +2485,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_OrganizationTable") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_OrganizationTable", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by org_id for consistent lock ordering across pods to prevent deadlocks. for org_id, response_cost in sorted(org_list_transactions.items()): @@ -2350,6 +2497,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2367,7 +2516,9 @@ class DBSpendUpdateWriter: start_time = time.time() try: async with ( - _spend_update_tx(prisma_client, "LiteLLM_OrganizationMembership") as transaction, + _spend_update_tx( + prisma_client, "LiteLLM_OrganizationMembership", outer_transaction=outer_transaction + ) as transaction, transaction.batch_() as batcher, ): for key, response_cost in sorted(org_member_list_transactions.items()): @@ -2378,6 +2529,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2393,6 +2546,7 @@ class DBSpendUpdateWriter: await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Project", transactions=project_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_projecttable", where_field="project_id", n_retry_times=n_retry_times, @@ -2401,16 +2555,18 @@ class DBSpendUpdateWriter: ) if on_table_committed is not None: on_table_committed("project_list_transactions") - await DBSpendUpdateWriter._invalidate_project_caches( - project_ids=tuple(project_list_transactions or ()), - proxy_logging_obj=proxy_logging_obj, - ) + if outer_transaction is None: + await DBSpendUpdateWriter._invalidate_project_caches( + project_ids=tuple(project_list_transactions or ()), + proxy_logging_obj=proxy_logging_obj, + ) ### UPDATE TAG TABLE ### tag_list_transactions: Final = db_spend_update_transactions["tag_list_transactions"] await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Tag", transactions=tag_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_tagtable", where_field="tag_name", n_retry_times=n_retry_times, @@ -2427,6 +2583,7 @@ class DBSpendUpdateWriter: await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Model access group", transactions=model_access_group_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_modelaccessgroupbudgettable", where_field="access_group_name", n_retry_times=n_retry_times, @@ -2441,6 +2598,7 @@ class DBSpendUpdateWriter: await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Agent", transactions=agent_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_agentstable", where_field="agent_id", n_retry_times=n_retry_times, @@ -2470,6 +2628,7 @@ class DBSpendUpdateWriter: n_retry_times: int, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, + outer_transaction: _SpendTransaction | None = None, ): """ Helper function to update spend for any entity type (team, org, tag, etc). @@ -2482,13 +2641,16 @@ class DBSpendUpdateWriter: n_retry_times: Number of retries on failure prisma_client: Prisma client instance proxy_logging_obj: Proxy logging object + outer_transaction: Existing ledger transaction owning the complete delivery update """ verbose_proxy_logger.debug("%s Spend transactions: %s", entity_name, transactions) if transactions is not None and len(transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, _ENTITY_SPEND_MODELS[table_accessor]) as transaction: + async with _spend_update_tx( + prisma_client, _ENTITY_SPEND_MODELS[table_accessor], outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks. for entity_id, response_cost in sorted(transactions.items()): @@ -2505,6 +2667,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await DBSpendUpdateWriter.handle_spend_update_failure( e=e, attempt=i, @@ -2838,6 +3002,12 @@ class DBSpendUpdateWriter: entity_id_field="tag", ) + def _record_request_error_safely(self, *, payload: SpendLogsPayload, prisma_client: PrismaClient | None) -> None: + try: + self._record_request_error(payload=payload, prisma_client=prisma_client) + except Exception as error: # noqa: BLE001 # error rollup failure must not skip sibling writes + _report_spend_update_step_failure("_record_request_error", payload=payload, exc=error) + @staticmethod def _record_request_error(*, payload: SpendLogsPayload, prisma_client: PrismaClient | None) -> None: if prisma_client is None: diff --git a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py index e9ec040023c..abbeab7f89d 100644 --- a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py +++ b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py @@ -55,3 +55,10 @@ class ToolDiscoveryQueue: """Return and clear all pending rows.""" items, self._pending = list(self._pending.values()), {} return items + + def requeue(self, items: Iterable[ToolDiscoveryQueueItem]) -> None: + """Put rows from a failed write back, ahead of and merged with anything queued since the flush.""" + since_flush: Final = self._pending + self._pending = {} + for item in (*items, *since_flush.values()): + self.add_update(item) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 8fbc8708f8c..c2d58512fea 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -31,10 +31,29 @@ _CONNECTION_CAPACITY_PHRASES: Final = ( "too many connections for database", "remaining connection slots are reserved", ) +_NOT_ACCEPTING_CONNECTIONS_SQLSTATE: Final = "57P03" +_NOT_ACCEPTING_CONNECTIONS_PHRASES: Final = ( + "the database system is starting up", + "the database system is shutting down", + "the database system is not yet accepting connections", + "the database system is not accepting connections", + "the database system is in recovery mode", +) +_SERVER_TERMINATED_SQLSTATES: Final = frozenset({"57P01", "57P02", "57P05"}) +_SERVER_TERMINATED_PHRASES: Final = ( + "terminating connection due to administrator command", + "terminating connection due to immediate shutdown command", + "terminating connection because of unexpected sigquit signal", + "terminating connection because of crash of another server process", + "terminating connection due to idle-session timeout", +) +_UNREACHABLE_PRISMA_CODE: Final = "P1001" +_UNREACHABLE_PHRASES: Final = ("can't reach database server", "cannot reach database server") _PRISMA_POOL_TIMEOUT_CODE: Final = "P2024" _LOCK_TIMEOUT_SQLSTATE: Final = "55P03" _LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout" _STATEMENT_TIMEOUT_SQLSTATE: Final = "57014" +_REJECTED_ROW_SQLSTATE_CLASSES: Final = frozenset({"22", "23", "54"}) _STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout" @@ -139,6 +158,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return True + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection @staticmethod @@ -201,6 +222,20 @@ class PrismaDBExceptionHandler: return type(e) is prisma.errors.DataError + @staticmethod + def is_rejected_row_error(e: Exception) -> bool: + """True iff Postgres refused the rows themselves: a base ``DataError``, or a raw query failing with a data + exception (SQLSTATE class 22), integrity violation (class 23) or a value too large for an index (class 54). Systemic raw failures such as a missing + table (42P01) or a deadlock (40P01) stay False so the batch is retried instead of dropped.""" + import prisma + + if PrismaDBExceptionHandler.is_prisma_data_error(e): + return True + if not isinstance(e, _exception_types(prisma.errors.RawQueryError)): + return False + sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) + return sqlstate is not None and sqlstate[:2] in _REJECTED_ROW_SQLSTATE_CLASSES + @staticmethod def is_database_transport_error(e: Exception) -> bool: """ @@ -289,6 +324,39 @@ class PrismaDBExceptionHandler: error_message: Final = str(e).lower() return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES) + @staticmethod + def is_database_not_accepting_connections_error(e: Exception) -> bool: + """True iff Postgres refused the connection because it is starting up, shutting + down or in crash recovery (SQLSTATE 57P03). The refusal comes before any statement + runs, so nothing the write carried was applied.""" + import prisma + + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) == _NOT_ACCEPTING_CONNECTIONS_SQLSTATE: + return True + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _NOT_ACCEPTING_CONNECTIONS_PHRASES) + + @staticmethod + def is_database_lifecycle_error(e: Exception) -> bool: + """True iff Postgres failed the statement because of a server lifecycle event: + a refused connection during startup, shutdown or recovery (57P03), or a session + the server terminated (57P01, 57P02, 57P05). The server comes back on its own, + so the rows are not at fault. A terminated session may have committed before it + was cut, so only ``is_database_not_accepting_connections_error`` proves a + non-idempotent write never applied.""" + import prisma + + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) in _SERVER_TERMINATED_SQLSTATES: + return True + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _SERVER_TERMINATED_PHRASES) + @staticmethod def is_lock_timeout_error(e: Exception) -> bool: """True iff Postgres cancelled a statement for exceeding ``lock_timeout`` @@ -406,6 +474,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return True + if PrismaDBExceptionHandler.is_database_lifecycle_error(e): + return True if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e): return True if "cached plan must not change result type" in str(e).lower(): @@ -680,3 +750,18 @@ def db_write_failure_is_requeue_safe(e: Exception) -> bool: return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES + + +def db_write_never_sent(e: Exception) -> bool: + """Whether the database could not take the write at all, because it was unreachable (P1001), + refusing connections (57P03) or full (53300, P2024), so even a non-idempotent write can be sent again.""" + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True + if PrismaDBExceptionHandler.is_database_capacity_error(e): + return True + if _prisma_error_code(e) == _UNREACHABLE_PRISMA_CODE: + return True + if not PrismaDBExceptionHandler.is_prisma_error(e): + return False + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _UNREACHABLE_PHRASES) diff --git a/litellm/proxy/db/model_usage_rollup.py b/litellm/proxy/db/model_usage_rollup.py index e6fbc299504..f8cdabe9296 100644 --- a/litellm/proxy/db/model_usage_rollup.py +++ b/litellm/proxy/db/model_usage_rollup.py @@ -32,14 +32,15 @@ class _UpsertTable(Protocol): def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... -class _ModelUsageBatch(Protocol): - litellm_dailymodelusage: _UpsertTable +class ModelUsageBatch(Protocol): + @property + def litellm_dailymodelusage(self) -> _UpsertTable: ... def execute_raw(self, query: LiteralString, *args: object) -> None: ... -class _ModelUsageBatchManager(Protocol): - async def __aenter__(self) -> _ModelUsageBatch: ... +class ModelUsageBatchManager(Protocol): + async def __aenter__(self) -> ModelUsageBatch: ... async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... @@ -116,8 +117,8 @@ def build_model_usage_transaction(payload: SpendLogsPayload) -> ModelUsageTransa ) -def _model_usage_batch(prisma_client: PrismaClient) -> _ModelUsageBatchManager: - batch: Final[_ModelUsageBatchManager] = prisma_client.db.batch_() +def _model_usage_batch(prisma_client: PrismaClient) -> ModelUsageBatchManager: + batch: Final[ModelUsageBatchManager] = prisma_client.db.batch_() return batch @@ -156,26 +157,16 @@ async def flush_model_usage_transactions( "custom_llm_provider": key.custom_llm_provider, "task_type": key.task_type, } - batcher.litellm_dailymodelusage.upsert( - where={"date_model_group_model_custom_llm_provider_task_type": key_fields}, - data={ - "create": { - **key_fields, - "spend": spend, - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "request_count": len(requests), - "successful_requests": successful, - "failed_requests": failed, - }, - "update": { - "spend": {"increment": spend}, - "prompt_tokens": {"increment": prompt_tokens}, - "completion_tokens": {"increment": completion_tokens}, - "request_count": {"increment": len(requests)}, - "successful_requests": {"increment": successful}, - "failed_requests": {"increment": failed}, - }, + queue_model_usage_upsert( + batcher, + key_fields, + { + "spend": spend, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "request_count": len(requests), + "successful_requests": successful, + "failed_requests": failed, }, ) return @@ -183,3 +174,15 @@ async def flush_model_usage_transactions( if attempt >= n_retry_times: raise await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) + + +def queue_model_usage_upsert( + batcher: ModelUsageBatch, key_fields: Mapping[str, str], increments: Mapping[str, int | float] +) -> None: + batcher.litellm_dailymodelusage.upsert( + where={"date_model_group_model_custom_llm_provider_task_type": key_fields}, + data={ + "create": {**key_fields, **increments}, + "update": {name: {"increment": value} for name, value in increments.items()}, + }, + ) diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py index f7ec88945a1..9600fb40ad3 100644 --- a/litellm/proxy/db/tool_registry_writer.py +++ b/litellm/proxy/db/tool_registry_writer.py @@ -23,7 +23,11 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, aggregate_tool_discoveries, ) -from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe +from litellm.proxy.db.exception_handler import ( + call_with_db_reconnect_retry, + db_write_failure_is_requeue_safe, + db_write_never_sent, +) from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions @@ -134,8 +138,19 @@ async def flush_tool_discovery_queue(prisma_client: "PrismaClient", tool_discove return try: await batch_upsert_tools(prisma_client=prisma_client, items=items) - except Exception as e: # noqa: BLE001 # a lost tool row only delays its discovery, the scheduler job must not fail - verbose_proxy_logger.debug("flush_tool_discovery_queue error (non-blocking): %s", e) + except Exception as e: # noqa: BLE001 # the rows go back to the queue or are dropped, the scheduler job must not fail + if db_write_never_sent(e): + tool_discovery_queue.requeue(items) + verbose_proxy_logger.warning( + "Tool discovery - %d tool rows went back to the queue for the next tick. Error: %s", len(items), str(e) + ) + return + verbose_proxy_logger.error( + "Tool discovery - dropped %d tool rows: the failed statement may have applied or the database " + "refused the data, so re-sending it is not safe. Error: %s", + len(items), + str(e), + ) async def commit_tool_discoveries( diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 8f929119dee..323cf57dfec 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -5,11 +5,13 @@ from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.gateway_activity import GatewayActivityStore from litellm.models.plugin_upload import PluginUploadStore +from litellm.models.skill_submission import SkillSubmissionStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository +from litellm.repositories.skill_submission_repository import SkillSubmissionRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -43,3 +45,12 @@ async def get_plugin_upload_repository(database: DatabaseDep) -> PluginUploadSto PluginUploadRepositoryDep: TypeAlias = Annotated[PluginUploadStore | None, Depends(get_plugin_upload_repository)] + + +async def get_skill_submission_repository(database: DatabaseDep) -> SkillSubmissionStore | None: + return None if database is None else SkillSubmissionRepository(database.prisma) + + +SkillSubmissionRepositoryDep: TypeAlias = Annotated[ + SkillSubmissionStore | None, Depends(get_skill_submission_repository) +] diff --git a/litellm/proxy/discovery_endpoints/agent_skills_archive.py b/litellm/proxy/discovery_endpoints/agent_skills_archive.py index dd78896725f..e45ca7e10f2 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_archive.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_archive.py @@ -11,26 +11,63 @@ import io import lzma import re import stat +import struct import zipfile import zlib -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final +from typing import ClassVar, Final import yaml -from pydantic import TypeAdapter +from pydantic import ConfigDict, TypeAdapter, ValidationError MAX_ARCHIVE_UNPACKED_BYTES: Final = 50 * 1024 * 1024 MAX_ARCHIVE_ENTRIES: Final = 1000 +MAX_SKILL_FILE_BYTES: Final = 10 * 1024 * 1024 +MAX_SKILL_MANIFEST_BYTES: Final = 1024 * 1024 +SKILL_ARCHIVE_ERRORS: Final = ( + zipfile.BadZipFile, + OSError, + RuntimeError, + EOFError, + lzma.LZMAError, + zlib.error, + struct.error, + UnicodeError, +) SKILL_MANIFEST_FILENAME: Final = "SKILL.md" _ZIP_ENTRY_TIMESTAMP: Final = (1980, 1, 1, 0, 0, 0) _ZIP_ENTRY_MODE: Final = 0o644 _FRONTMATTER_PATTERN: Final = re.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", re.DOTALL) +_AGENT_SKILL_FRONTMATTER_PATTERN: Final = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\Z)", re.DOTALL) +_AGENT_SKILL_YAML_SCALARS: Final = MappingProxyType( + { + "tag:yaml.org,2002:null": re.compile(r"^(?:null|Null|NULL|~|)$"), + "tag:yaml.org,2002:bool": re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), + "tag:yaml.org,2002:int": re.compile(r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$"), + "tag:yaml.org,2002:float": re.compile( + r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$" + ), + } +) +_MAX_FRONTMATTER_DEPTH: Final = 32 _WINDOWS_DRIVE_PATTERN: Final = re.compile(r"^[A-Za-z]:") _EMPTY_FRONTMATTER: Final[Mapping[str, object]] = MappingProxyType({}) _DECODED_YAML: Final = TypeAdapter(object) +_YAML_EVENTS: Final = TypeAdapter(Iterable[object]) +_YAML_NODES: Final = TypeAdapter(tuple[yaml.Node, ...], config=ConfigDict(arbitrary_types_allowed=True)) +_YAML_PAIRS: Final = TypeAdapter( + tuple[tuple[yaml.Node, yaml.Node], ...], config=ConfigDict(arbitrary_types_allowed=True) +) + + +class _AgentSkillFrontmatterLoader(yaml.BaseLoader): + yaml_implicit_resolvers: ClassVar[Mapping[str | None, Sequence[tuple[str, re.Pattern[str]]]]] = { # pyright: ignore[reportIncompatibleVariableOverride] # PyYAML resolver tables are class state + None: list(_AGENT_SKILL_YAML_SCALARS.items()) + } @dataclass(frozen=True, slots=True) @@ -39,37 +76,100 @@ class SkillArchive: digest: str declared_name: str | None declared_description: str | None + manifest_content: bytes def read_skill_archive_members( - stored_content: bytes, *, for_upload: bool = False + stored_content: bytes, *, for_upload: bool = False, skill_path: str | None = None ) -> tuple[tuple[str, bytes, int], ...] | None: try: with zipfile.ZipFile(io.BytesIO(stored_content)) as uploaded: - return _flattened_members(uploaded, for_upload=for_upload) - except (zipfile.BadZipFile, OSError, RuntimeError, EOFError, lzma.LZMAError, zlib.error): + return _flattened_members(uploaded, for_upload=for_upload, skill_path=skill_path) + except SKILL_ARCHIVE_ERRORS: return None +def read_skill_archive_entry(uploaded: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes: + with uploaded.open(info) as member: + return member.read(info.file_size + 1) + + def build_skill_archive(stored_content: bytes) -> SkillArchive | None: """Return the installable archive for an upload, or None when it holds no root SKILL.md.""" members: Final = read_skill_archive_members(stored_content) if members is None: return None - frontmatter: Final = _manifest_frontmatter( - next(data for name, data, _ in members if name == SKILL_MANIFEST_FILENAME) - ) + manifest_content: Final = next(data for name, data, _ in members if name == SKILL_MANIFEST_FILENAME) + frontmatter: Final = _manifest_frontmatter(manifest_content) content: Final = pack_skill_archive(members) return SkillArchive( content=content, digest=f"sha256:{hashlib.sha256(content).hexdigest()}", declared_name=_frontmatter_text(frontmatter, "name"), declared_description=_frontmatter_text(frontmatter, "description"), + manifest_content=manifest_content, ) -def _flattened_members(uploaded: zipfile.ZipFile, *, for_upload: bool) -> tuple[tuple[str, bytes, int], ...] | None: +def build_agent_skill_archive( + content: bytes, *, skill_path: str | None = None, plugin_name: str | None = None +) -> SkillArchive | None: + members: Final = read_skill_archive_members(content, for_upload=True, skill_path=skill_path) + selected: Final = ( + members + if members is not None or plugin_name is None + else read_skill_archive_members(content, for_upload=True, skill_path=f"skills/{plugin_name}") + ) + if selected is None: + return None + manifest_content: Final = next(data for path, data, _ in selected if path == SKILL_MANIFEST_FILENAME) + manifest: Final = agent_skill_frontmatter(manifest_content) + name: Final = _frontmatter_text(manifest, "name") + description: Final = _frontmatter_text(manifest, "description") + if ( + name is None + or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name) + or len(name) > 64 + or description is None + or len(description) > 1024 + ): + return None + archive: Final = pack_skill_archive(selected) + return SkillArchive( + content=archive, + digest=f"sha256:{hashlib.sha256(archive).hexdigest()}", + declared_name=name, + declared_description=description, + manifest_content=manifest_content, + ) + + +def _flattened_members( + uploaded: zipfile.ZipFile, *, for_upload: bool, skill_path: str | None +) -> tuple[tuple[str, bytes, int], ...] | None: + kept: Final = skill_archive_entries(uploaded, for_upload=for_upload) + if kept is None: + return None + prefix: Final = f"{skill_path}/" if skill_path is not None else _common_root_prefix(tuple(path for _, path in kept)) + flattened: Final = tuple((info, path[len(prefix) :]) for info, path in kept if path.startswith(prefix)) + names: Final = frozenset(name for _, name in flattened) + if ( + SKILL_MANIFEST_FILENAME not in names + or len(names) != len(flattened) + or (for_upload and any(not safe_skill_archive_entry(info) for info, _ in flattened)) + ): + return None + + return tuple( + (name, read_skill_archive_entry(uploaded, info), _entry_mode(info, preserve_executable_bits=for_upload)) + for info, name in sorted(flattened, key=lambda member: member[1]) + ) + + +def skill_archive_entries( + uploaded: zipfile.ZipFile, *, for_upload: bool +) -> tuple[tuple[zipfile.ZipInfo, str], ...] | None: infos: Final = tuple(info for info in uploaded.infolist() if for_upload or not info.is_dir()) if not infos or len(infos) > MAX_ARCHIVE_ENTRIES: return None @@ -81,26 +181,13 @@ def _flattened_members(uploaded: zipfile.ZipFile, *, for_upload: bool) -> tuple[ ) if any(path is None for _, path in normalized): return None - if for_upload and ( - len(frozenset(path for _, path in normalized)) != len(normalized) - or any(not _safe_upload_entry(info) for info in infos) - ): - return None - - kept: Final = tuple( - (info, path) - for info, path in normalized - if path is not None and not info.is_dir() and (not for_upload or not _is_macos_metadata(path)) - ) - prefix: Final = _common_root_prefix(tuple(path for _, path in kept)) - flattened: Final = tuple((info, path[len(prefix) :]) for info, path in kept) - names: Final = frozenset(name for _, name in flattened) - if SKILL_MANIFEST_FILENAME not in names or len(names) != len(flattened): + if for_upload and len(frozenset(path for _, path in normalized)) != len(normalized): return None return tuple( - (name, uploaded.read(info), _entry_mode(info, preserve_executable_bits=for_upload)) - for info, name in sorted(flattened, key=lambda member: member[1]) + (info, path) + for info, path in normalized + if path is not None and not info.is_dir() and (not for_upload or not _is_macos_metadata(path)) ) @@ -113,9 +200,11 @@ def _entry_mode(info: zipfile.ZipInfo, *, preserve_executable_bits: bool) -> int return _ZIP_ENTRY_MODE | (mode & 0o111) -def _safe_upload_entry(info: zipfile.ZipInfo) -> bool: - return not info.flag_bits & 1 and ( - info.create_system != 3 or stat.S_IFMT(info.external_attr >> 16) in (0, stat.S_IFREG, stat.S_IFDIR) +def safe_skill_archive_entry(info: zipfile.ZipInfo) -> bool: + return ( + info.file_size <= MAX_SKILL_FILE_BYTES + and not info.flag_bits & 1 + and (info.create_system != 3 or stat.S_IFMT(info.external_attr >> 16) in (0, stat.S_IFREG, stat.S_IFDIR)) ) @@ -154,6 +243,65 @@ def _manifest_frontmatter(manifest: bytes) -> Mapping[str, object]: return parsed +def agent_skill_frontmatter(manifest: bytes) -> Mapping[str, str]: + if len(manifest) > MAX_SKILL_MANIFEST_BYTES: + return MappingProxyType({}) + try: + match: Final = _AGENT_SKILL_FRONTMATTER_PATTERN.match(manifest.decode("utf-8-sig")) + if match is None: + return MappingProxyType({}) + frontmatter: Final = match.group(1) + events: Final = _YAML_EVENTS.validate_python(yaml.parse(frontmatter, Loader=yaml.BaseLoader)) # pyright: ignore[reportUnknownMemberType] # PyYAML parse stubs omit their event iterator return type + if any(isinstance(event, yaml.AliasEvent) for event in events): + return MappingProxyType({}) + node: Final = _DECODED_YAML.validate_python(yaml.compose(frontmatter, Loader=_AgentSkillFrontmatterLoader)) # pyright: ignore[reportUnknownMemberType] # PyYAML compose stubs omit their return type + except (yaml.YAMLError, UnicodeError, RecursionError, ValidationError): + return MappingProxyType({}) + if not isinstance(node, yaml.MappingNode) or not _valid_frontmatter_node(node): + return MappingProxyType({}) + pairs: Final = _YAML_PAIRS.validate_python(node.value) + fields: Final = { + key_text: value_text + for key, value in pairs + if (key_text := _frontmatter_string(key)) is not None and (value_text := _frontmatter_string(value)) is not None + } + if "name" not in fields or "description" not in fields: + return MappingProxyType({}) + return MappingProxyType(fields) + + +def _frontmatter_string(node: yaml.Node) -> str | None: + if not isinstance(node, yaml.ScalarNode) or node.tag != "tag:yaml.org,2002:str": + return None + value: Final = _DECODED_YAML.validate_python(node.value) + return value if isinstance(value, str) else None + + +def _valid_frontmatter_node(node: yaml.Node, depth: int = 0) -> bool: + if depth > _MAX_FRONTMATTER_DEPTH: + return False + if isinstance(node, yaml.ScalarNode): + pattern: Final = _AGENT_SKILL_YAML_SCALARS.get(node.tag) + scalar: Final = _DECODED_YAML.validate_python(node.value) + return isinstance(scalar, str) and ( + node.tag == "tag:yaml.org,2002:str" or (pattern is not None and pattern.fullmatch(scalar) is not None) + ) + if isinstance(node, yaml.SequenceNode): + values: Final = _YAML_NODES.validate_python(node.value) + return node.tag == "tag:yaml.org,2002:seq" and all( + _valid_frontmatter_node(value, depth + 1) for value in values + ) + if not isinstance(node, yaml.MappingNode) or node.tag != "tag:yaml.org,2002:map": + return False + pairs: Final = _YAML_PAIRS.validate_python(node.value) + keys: Final = tuple(_frontmatter_string(key) for key, _ in pairs) + return ( + None not in keys + and len(frozenset(keys)) == len(keys) + and all(_valid_frontmatter_node(value, depth + 1) for _, value in pairs) + ) + + def _frontmatter_text(frontmatter: Mapping[str, object], key: str) -> str | None: value: Final = frontmatter.get(key) if not isinstance(value, str): diff --git a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py index 830f7b0334e..3b589b1a4f0 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -9,7 +9,7 @@ is enabled, which publishes every stored skill to anyone who can reach the proxy import asyncio import re from collections.abc import Sequence -from itertools import groupby +from itertools import count from types import MappingProxyType from typing import Final @@ -19,6 +19,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.models.skills import LiteLLM_SkillsTable +from litellm.proxy.deps.repositories import PluginUploadRepositoryDep from litellm.proxy.discovery_endpoints.agent_skills_archive import SkillArchive, build_skill_archive from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( MAX_SKILL_DESCRIPTION_LENGTH, @@ -83,28 +84,34 @@ async def stored_skill(skill_id: str) -> LiteLLM_SkillsTable | None: ) async def agent_skills_index( request: Request, + uploads: PluginUploadRepositoryDep, skills: Sequence[LiteLLM_SkillsTable] = Depends(stored_skills), ) -> AgentSkillsIndex: - """Agent Skills v0.2.0 discovery index over every skill stored on this proxy.""" + """Discover public Hub skills and the separately enabled hosted skill store.""" + from litellm.proxy.discovery_endpoints.skill_hub_endpoints import public_hub_entries from litellm.proxy.utils import get_custom_url - installable: Final = await _installable(skills) + installable: Final = await _installable(skills[:MAX_INDEXED_SKILLS]) names: Final = _deduplicated(tuple(_base_name(skill, archive) for skill, archive in installable)) - - return AgentSkillsIndex( - skills=tuple( - AgentSkillsIndexEntry( - name=name, - type="archive", - description=_description(skill, archive, name), - url=get_custom_url( - request_base_url=str(request.base_url), - route=f"v1/skills/{skill.skill_id}/archive", - ), - digest=archive.digest, - ) - for (skill, archive), name in zip(installable, names, strict=True) + legacy: Final = tuple( + AgentSkillsIndexEntry( + name=name, + type="archive", + description=_description(skill, archive, name), + url=get_custom_url( + request_base_url=str(request.base_url), + route=f"v1/skills/{skill.skill_id}/archive", + ), + digest=archive.digest, ) + for (skill, archive), name in zip(installable, names, strict=True) + ) + remaining: Final = MAX_INDEXED_SKILLS - len(legacy) + hub: Final = (await public_hub_entries(request, uploads))[:remaining] if remaining else () + hub_names: Final = _deduplicated(tuple(entry.name for entry in hub), reserved=frozenset(names)) + return AgentSkillsIndex( + skills=legacy + + tuple(entry.model_copy(update={"name": name}) for entry, name in zip(hub, hub_names, strict=True)) ) @@ -153,7 +160,7 @@ async def _archive_for(skill: LiteLLM_SkillsTable) -> SkillArchive | None: ) return None - if cache_key is not None and len(archive.content) <= MAX_CACHED_ARCHIVE_BYTES: + if cache_key is not None and len(archive.content) + len(archive.manifest_content) <= MAX_CACHED_ARCHIVE_BYTES: _ARCHIVE_CACHE.set_cache(cache_key, archive) return archive @@ -172,19 +179,25 @@ def _slugify(raw: str | None) -> str | None: return _NON_SLUG_PATTERN.sub("-", raw.lower()).strip("-")[:MAX_SKILL_NAME_LENGTH].rstrip("-") or None -def _entry_name(entry: tuple[int, str]) -> str: - return entry[1] +def _deduplicated(names: Sequence[str], reserved: frozenset[str] = frozenset()) -> tuple[str, ...]: + natural_names: Final = frozenset(names) | reserved + ordinals: Final = MappingProxyType({name: count(1) for name in frozenset(names)}) + used: Final[set[str]] = set(reserved) # mutable-ok: bounded local allocation avoids rebuilding the set + def allocate(name: str) -> str: + selected: Final = ( + name + if name not in used + else next( + candidate + for candidate in (_with_ordinal(name, ordinal) for ordinal in ordinals[name]) + if candidate not in natural_names and candidate not in used + ) + ) + used.add(selected) + return selected -def _deduplicated(names: Sequence[str]) -> tuple[str, ...]: - ordinals: Final = MappingProxyType( - { - position: ordinal - for _, duplicates in groupby(sorted(enumerate(names), key=_entry_name), key=_entry_name) - for ordinal, (position, _) in enumerate(duplicates) - } - ) - return tuple(_with_ordinal(name, ordinals[position]) for position, name in enumerate(names)) + return tuple(allocate(name) for name in names) def _with_ordinal(name: str, ordinal: int) -> str: diff --git a/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py b/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py new file mode 100644 index 00000000000..93d1ace435d --- /dev/null +++ b/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py @@ -0,0 +1,682 @@ +import asyncio +import hashlib +from collections.abc import Mapping +from dataclasses import dataclass, replace +from datetime import datetime +from typing import Annotated, Final +from urllib.parse import quote + +import httpx +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response, UploadFile + +import litellm +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory has untyped optional params +) +from litellm.models.plugin_upload import ( + PluginArchiveMetadata, + PluginDownloader, + PluginDownloadsResponse, + PluginPublication, + PluginSnapshot, + PluginUploadStore, + PluginVersion, + PublishedPlugin, + has_agent_skill_metadata, +) +from litellm.models.skill_submission import SkillSubmission, SkillSubmissionStore +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( + MANIFEST_FIELDS, + delete_plugin, + disable_plugin, + enable_plugin, + get_plugin_downloads, + header_key_caller, + list_plugins, + plugin_archive_source_matches, + plugin_archive_url, + plugin_downloader, + require_download_recorded, +) +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import resolve_skill_visibility +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.skill_upload_archive import build_uploaded_skill_archive +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.claude_code_marketplace_auth import ( + marketplace_auth_required, + marketplace_key_header_present, +) +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin +from litellm.proxy.deps.repositories import PluginUploadRepositoryDep, SkillSubmissionRepositoryDep +from litellm.proxy.discovery_endpoints.agent_skills_archive import ( + SkillArchive, + build_agent_skill_archive, +) +from litellm.proxy.discovery_endpoints.skill_hub_mutations import ( + SkillMutationError, + archive_writer, + write_skill, + write_target, +) +from litellm.proxy.discovery_endpoints.skill_import import ( + MAX_IMPORT_BYTES, + SkillImportError, + fetch_skill_source, + parse_skill_source, + select_imported_skill, +) +from litellm.types.proxy.claude_code_endpoints import PluginListItem +from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import AgentSkillsIndexEntry +from litellm.types.proxy.skill_hub import ( + ImportSkillRequest, + SkillHubList, + SkillHubMetadata, + SkillHubMutationResponse, + SkillHubPendingSkill, + SkillHubProposal, + SkillHubPublication, + SkillHubSkill, + SkillHubVersion, + SkillHubVersions, +) + +router: Final = APIRouter(prefix="/skill-hub/skills", tags=["Skills Hub"]) +_EDITABLE_METADATA: Final = frozenset(SkillHubMetadata.model_fields) +router.add_api_route( + "/{plugin_name}/downloads", get_plugin_downloads, methods=["GET"], response_model=PluginDownloadsResponse +) + + +@dataclass(frozen=True, slots=True) +class VisibleSkillArchive: + archive: SkillArchive + metadata: PluginArchiveMetadata + downloader: PluginDownloader + + +def _url(request: Request, route: str) -> str: + from litellm.proxy.utils import get_custom_url + + return get_custom_url(request_base_url=str(request.base_url), route=route.lstrip("/")) + + +def _download_url(request: Request, name: str) -> str: + return _url(request, f"skill-hub/skills/{quote(name, safe='')}/archive") + + +def _skill_response(request: Request, plugin: PluginListItem, download_count: int | None = None) -> SkillHubSkill: + local_archive: Final = plugin_archive_source_matches( + request, + plugin.name, + plugin.source, + plugin.source.get("sha256", ""), + has_agent_skill_metadata=plugin.has_agent_skill_metadata, + ) + return SkillHubSkill( + id=plugin.id, + name=plugin.name, + description=plugin.description, + version=plugin.version, + domain=plugin.domain, + namespace=plugin.namespace, + keywords=None if plugin.keywords is None else tuple(plugin.keywords), + updated_at=None if plugin.updated_at is None else datetime.fromisoformat(plugin.updated_at), + visibility="public" if plugin.enabled else "private", + source={"source": "archive", "url": _download_url(request, plugin.name)} if local_archive else plugin.source, + download_url=_download_url(request, plugin.name) if local_archive else None, + content_url=_url(request, f"skill-hub/skills/{quote(plugin.name, safe='')}/content") if local_archive else None, + download_count=download_count, + ) + + +@router.get("", response_model=SkillHubList) +async def list_skills( + request: Request, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + query: str = "", + offset: Annotated[int, Query(ge=0)] = 0, + limit: Annotated[int, Query(ge=1, le=100)] = 50, +) -> SkillHubList: + """Browse the caller's public and granted private skills.""" + catalog: Final = await list_plugins(enabled_only=False, user_api_key_dict=caller) + needle: Final = query.casefold() + matched: Final = tuple( + plugin + for plugin in sorted(catalog.plugins, key=lambda item: item.name) + if needle in " ".join((plugin.name, plugin.description or "", *(plugin.keywords or ()))).casefold() + ) + page: Final = matched[offset : offset + limit] + counts: Final[Mapping[str, int]] = ( + await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page)) + if is_proxy_admin(caller) + else {} + ) + return SkillHubList( + skills=tuple(_skill_response(request, plugin, counts.get(plugin.id)) for plugin in page), + count=len(matched), + next_offset=offset + limit if offset + limit < len(matched) else None, + ) + + +@router.get("/{name}", response_model=SkillHubSkill) +async def get_skill( + request: Request, + name: str, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubSkill: + """Read one visible skill and its portable download URL.""" + store: Final = _require_store(uploads) + metadata: Final = await store.get_metadata(name) + if metadata is None or not (await resolve_skill_visibility(caller)).allows(metadata): + raise HTTPException(404, "Skill not found") + return skill_metadata_response(request, metadata) + + +def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -> SkillHubSkill: + manifest: Final = MANIFEST_FIELDS.validate_json(metadata.manifest_json or "{}") + return _skill_response( + request, + PluginListItem.model_validate( + { + "id": metadata.id or metadata.name, + "name": metadata.name, + "description": metadata.description, + "version": metadata.version, + "source": dict(metadata.source), + "has_agent_skill_metadata": metadata.has_agent_skill_metadata, + "enabled": metadata.enabled, + "created_at": None, + "updated_at": metadata.updated_at.isoformat() if metadata.updated_at is not None else None, + "domain": manifest.get("domain"), + "namespace": manifest.get("namespace"), + "keywords": manifest.get("keywords"), + } + ), + ) + + +def _require_store(uploads: PluginUploadStore | None) -> PluginUploadStore: + if uploads is None: + raise HTTPException(503, "Skills Hub requires a connected database") + return uploads + + +def _require_admin(caller: UserAPIKeyAuth) -> None: + if not is_proxy_admin(caller): + raise HTTPException(403, "Only proxy admins may publish skills") + + +@router.put("/{name}", response_model=SkillHubMutationResponse) +async def replace_skill_metadata( + request: Request, + name: str, + body: SkillHubMetadata, + uploads: PluginUploadRepositoryDep, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubMutationResponse: + """Replace registry metadata, preserving files. Members submit an immutable proposal for admin review.""" + existing: Final = await write_target(name, True, uploads, caller, body.expected_updated_at) + if isinstance(existing, SkillMutationError): + raise HTTPException(existing.status, existing.message) + if existing is None: + raise HTTPException(404, "Skill not found") + publication: Final = PluginPublication( + name=name, + version=body.version, + description=body.description, + manifest={ + **{key: value for key, value in existing.manifest.items() if key not in _EDITABLE_METADATA}, + **body.model_dump(exclude_none=True), + "name": name, + }, + created_by=caller.user_id, + ) + return await _mutate(request, publication, existing, _require_store(uploads), submissions, caller) + + +@router.post("/{name}/publish", response_model=SkillHubSkill) +async def publish_skill( + request: Request, + name: str, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + _body: SkillHubPublication | None = None, +) -> SkillHubSkill: + """Make a skill public using the existing Hub publication policy.""" + _require_admin(caller) + await enable_plugin(plugin_name=name, user_api_key_dict=caller) + return await get_skill(request, name, uploads, caller) + + +@router.post("/{name}/unpublish", response_model=SkillHubSkill) +async def unpublish_skill( + request: Request, + name: str, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + _body: SkillHubPublication | None = None, +) -> SkillHubSkill: + """Remove a skill from public discovery while retaining explicit grants.""" + _require_admin(caller) + await disable_plugin(plugin_name=name, user_api_key_dict=caller) + return await get_skill(request, name, uploads, caller) + + +@router.delete("/{name}", status_code=204, response_class=Response) +async def remove_skill(name: str, caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]) -> Response: + """Delete a skill and its managed archive.""" + _require_admin(caller) + await delete_plugin(plugin_name=name, user_api_key_dict=caller) + return Response(status_code=204) + + +@router.post("/upload", response_model=SkillHubMutationResponse) +async def upload_skill( + request: Request, + uploads: PluginUploadRepositoryDep, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(archive_writer)], + file: Annotated[UploadFile, File(description="Agent Skills ZIP with SKILL.md and optional supporting files")], + version: Annotated[str | None, Form()] = None, +) -> SkillHubMutationResponse: + """Publish a portable skill while retaining Claude marketplace compatibility.""" + store: Final = _require_store(uploads) + content: Final = await file.read(MAX_IMPORT_BYTES + 1) + if len(content) > MAX_IMPORT_BYTES: + raise HTTPException(413, "Skill ZIP must be 10 MiB or smaller") + archive: Final = await asyncio.to_thread(build_agent_skill_archive, content) + if archive is None: + raise HTTPException(422, "Upload a valid Agent Skill with name and description in SKILL.md") + return await _publish(request, store, submissions, caller, archive, version, None) + + +def import_http_client() -> httpx.AsyncClient: + return get_async_httpx_client(llm_provider="skill_hub_import").client + + +@router.post("/import", response_model=SkillHubMutationResponse) +async def import_skill( + request: Request, + body: ImportSkillRequest, + uploads: PluginUploadRepositoryDep, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(archive_writer)], + client: Annotated[httpx.AsyncClient, Depends(import_http_client)], +) -> SkillHubMutationResponse: + """Import a public skills.sh or GitHub skill as a portable snapshot.""" + store: Final = _require_store(uploads) + source: Final = parse_skill_source(body.url, skill=body.skill, ref=body.ref, path=body.path) + if isinstance(source, SkillImportError): + raise HTTPException(source.status, source.message) + content: Final = await fetch_skill_source(source, client) + if isinstance(content, SkillImportError): + raise HTTPException(content.status, content.message) + archive: Final = await asyncio.to_thread(select_imported_skill, content, source) + if isinstance(archive, SkillImportError): + raise HTTPException(archive.status, archive.message) + return await _publish(request, store, submissions, caller, archive, body.version, body.url) + + +async def _publish( + request: Request, + store: PluginUploadStore, + submissions: SkillSubmissionStore | None, + caller: UserAPIKeyAuth, + archive: SkillArchive, + version: str | None, + origin: str | None, +) -> SkillHubMutationResponse: + existing: Final = await write_target(archive.declared_name or "", False, store, caller) + if isinstance(existing, SkillMutationError): + raise HTTPException(existing.status, existing.message) + publication: Final = await prepare_skill_publication(request, archive, version, origin, caller.user_id) + return await _mutate(request, publication, existing, store, submissions, caller) + + +async def _mutate( + request: Request, + publication: PluginPublication, + existing: PluginSnapshot | None, + store: PluginUploadStore, + submissions: SkillSubmissionStore | None, + caller: UserAPIKeyAuth, +) -> SkillHubMutationResponse: + result: Final = await write_skill(publication, existing, store, submissions, caller) + if isinstance(result, SkillMutationError): + raise HTTPException(result.status, result.message) + if isinstance(result, SkillSubmission): + return SkillHubPendingSkill(submission_id=result.id, proposal=skill_proposal_response(request, result)) + return skill_publication_response(request, publication, result) + + +def skill_proposal_response(request: Request, submission: SkillSubmission) -> SkillHubProposal: + proposal: Final = submission.proposal + return SkillHubProposal.model_validate( + { + **proposal.manifest, + "name": proposal.name, + "description": proposal.description, + "version": proposal.version, + **( + { + "source": { + "source": "archive", + "url": _url(request, f"skill-hub/submissions/{quote(submission.id, safe='')}/archive"), + } + } + if submission.has_archive + else {} + ), + } + ) + + +@router.put("/{name}/upload", response_model=SkillHubMutationResponse) +async def replace_skill_archive( + request: Request, + name: str, + uploads: PluginUploadRepositoryDep, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(archive_writer)], + file: Annotated[UploadFile, File(description="Replacement Agent Skills ZIP with the same declared skill name")], + version: Annotated[str | None, Form()] = None, + expected_updated_at: Annotated[datetime | None, Form()] = None, +) -> SkillHubMutationResponse: + """Replace skill files, retaining registry fields. Members propose an update against the original editor timestamp.""" + existing: Final = await write_target(name, True, uploads, caller, expected_updated_at) + if isinstance(existing, SkillMutationError): + raise HTTPException(existing.status, existing.message) + if existing is None: + raise HTTPException(404, "Skill not found") + content: Final = await file.read(MAX_IMPORT_BYTES + 1) + if len(content) > MAX_IMPORT_BYTES: + raise HTTPException(413, "Skill ZIP must be 10 MiB or smaller") + archive: Final = await asyncio.to_thread(build_agent_skill_archive, content) + if archive is None or archive.declared_name != name: + raise HTTPException(422, "Replacement SKILL.md must declare the existing skill name and a description") + prepared: Final = await prepare_skill_publication(request, archive, version, None, caller.user_id) + publication: Final = replace(prepared, manifest={**existing.manifest, **prepared.manifest}) + return await _mutate(request, publication, existing, _require_store(uploads), submissions, caller) + + +def skill_publication_response( + request: Request, publication: PluginPublication, published: PublishedPlugin +) -> SkillHubSkill: + return _skill_response( + request, + PluginListItem.model_validate( + { + "source": publication.manifest.get("source", {}), + "id": published.id, + "name": published.name, + "description": publication.description, + "version": published.version, + "enabled": published.enabled, + "has_agent_skill_metadata": has_agent_skill_metadata(publication.manifest), + "created_at": None, + "updated_at": published.updated_at.isoformat(), + "domain": publication.manifest.get("domain"), + "namespace": publication.manifest.get("namespace"), + "keywords": publication.manifest.get("keywords"), + } + ), + ) + + +async def prepare_skill_publication( + request: Request, + archive: SkillArchive, + version: str | None, + origin: str | None, + created_by: str | None, +) -> PluginPublication: + name: Final = archive.declared_name + if name is None: + raise HTTPException(422, "SKILL.md must declare a skill name") + plugin: Final = await asyncio.to_thread(build_uploaded_skill_archive, archive.content, name) + if plugin is None: + raise HTTPException(422, "Skill ZIP exceeds the supported archive limits") + portable: Final = await asyncio.to_thread(build_agent_skill_archive, plugin, plugin_name=name) + if portable is None or portable.digest != archive.digest: + raise HTTPException(422, "Skill ZIP exceeds the supported archive limits after compatibility packaging") + digest: Final = hashlib.sha256(plugin).hexdigest() + source: Final = { + "source": "archive", + "url": plugin_archive_url(request, name, digest), + "sha256": digest, + } + manifest: Final = { + "name": name, + "description": archive.declared_description, + "version": version, + "source": source, + "agent_skill": { + "name": name, + "description": archive.declared_description, + "digest": archive.digest, + "origin": origin, + }, + } + return PluginPublication( + name=name, + version=version, + description=archive.declared_description, + manifest=manifest, + content=plugin, + created_by=created_by, + ) + + +async def _version_metadata(name: str, store: PluginUploadStore, caller: UserAPIKeyAuth) -> PluginArchiveMetadata: + metadata: Final = await store.get_metadata(name) + if metadata is None or metadata.id is None or not (await resolve_skill_visibility(caller)).allows(metadata): + raise HTTPException(404, "Skill not found") + return metadata + + +def _version_response(request: Request, name: str, version: PluginVersion) -> SkillHubVersion: + return SkillHubVersion( + id=version.id, + version=version.version, + description=version.description, + manifest=version.manifest, + published_at=version.published_at, + published_by=version.published_by, + content_kind="stored_archive" if version.archive_available else "source_reference", + download_url=( + _url(request, f"skill-hub/skills/{quote(name, safe='')}/versions/{quote(version.id, safe='')}/archive") + if version.archive_available + else None + ), + ) + + +@router.get("/{name}/versions", response_model=SkillHubVersions) +async def list_skill_versions( + request: Request, + name: str, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + limit: Annotated[int, Query(ge=1, le=100)] = 20, + offset: Annotated[int, Query(ge=0)] = 0, +) -> SkillHubVersions: + """Read published metadata history. External references are not retained artifacts.""" + store: Final = _require_store(uploads) + metadata: Final = await _version_metadata(name, store, caller) + versions: Final = await store.versions(metadata, limit + 1, offset) + return SkillHubVersions( + versions=tuple(_version_response(request, name, version) for version in versions[:limit]), + next_offset=offset + limit if len(versions) > limit else None, + ) + + +@router.get("/{name}/versions/{version_id}", response_model=SkillHubVersion) +async def get_skill_version( + request: Request, + name: str, + version_id: str, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubVersion: + """Read a retained publication under the current skill's visibility policy.""" + store: Final = _require_store(uploads) + metadata: Final = await _version_metadata(name, store, caller) + version: Final = await store.version(metadata, version_id) + if version is None: + raise HTTPException(404, "Skill version not found") + return _version_response(request, name, version) + + +@router.get( + "/{name}/versions/{version_id}/archive", + response_class=Response, + responses={200: {"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}}}}, +) +async def download_skill_version( + name: str, + version_id: str, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """Download a retained publication as a portable ZIP with SKILL.md at its root.""" + store: Final = _require_store(uploads) + metadata: Final = await _version_metadata(name, store, caller) + version: Final = await store.version(metadata, version_id) + if version is None: + raise HTTPException(404, "Skill version not found") + content: Final = await store.version_content(metadata, version_id) + archive: Final = ( + await asyncio.to_thread(build_agent_skill_archive, content, plugin_name=name) if content is not None else None + ) + if archive is None: + raise HTTPException(404, "This version has no retained Agent Skills archive") + require_download_recorded( + await store.record_download( + replace(metadata, version=version.version), + archive.digest.removeprefix("sha256:"), + plugin_downloader(caller), + ) + ) + return Response( + archive.content, + media_type="application/zip", + headers={ + "Content-Disposition": f'attachment; filename="{name}.zip"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "ETag": f'"{archive.digest}"', + }, + ) + + +async def _visible_archive(request: Request, name: str, uploads: PluginUploadStore | None) -> VisibleSkillArchive: + anonymous_allowed: Final = litellm.public_skills_index is True and not marketplace_auth_required() + caller: Final = ( + await header_key_caller(request) if marketplace_key_header_present(request) or not anonymous_allowed else None + ) + store: Final = _require_store(uploads) + metadata: Final = await store.get_metadata(name) + if metadata is None or not (await resolve_skill_visibility(caller)).allows(metadata): + raise HTTPException(404, "Skill not found") + if not plugin_archive_source_matches( + request, + name, + metadata.source, + metadata.source.get("sha256", ""), + has_agent_skill_metadata=metadata.has_agent_skill_metadata, + ): + raise HTTPException(404, "This skill no longer refers to a managed archive") + content: Final = await store.get_content(metadata) + archive: Final = ( + await asyncio.to_thread(build_agent_skill_archive, content, plugin_name=name) if content is not None else None + ) + if archive is None: + raise HTTPException(404, "This skill has no stored Agent Skills archive") + return VisibleSkillArchive(archive, metadata, plugin_downloader(caller)) + + +@router.get( + "/{name}/archive", + response_class=Response, + responses={200: {"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}}}}, +) +async def download_skill( + request: Request, name: str, uploads: PluginUploadRepositoryDep, digest: str | None = None +) -> Response: + """Download the complete portable skill, with SKILL.md at the ZIP root.""" + visible: Final = await _visible_archive(request, name, uploads) + archive: Final = visible.archive + if digest is not None and digest != archive.digest: + raise HTTPException(404, "Skill archive version not found") + require_download_recorded( + await _require_store(uploads).record_download( + visible.metadata, archive.digest.removeprefix("sha256:"), visible.downloader + ) + ) + return Response( + archive.content, + media_type="application/zip", + headers={ + "Content-Disposition": f'attachment; filename="{name}.zip"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "ETag": f'"{archive.digest}"', + }, + ) + + +@router.get("/{name}/content", response_class=Response, responses={200: {"content": {"text/markdown": {}}}}) +async def skill_content(request: Request, name: str, uploads: PluginUploadRepositoryDep) -> Response: + """Read the original SKILL.md without downloading supporting files.""" + visible: Final = await _visible_archive(request, name, uploads) + return Response( + visible.archive.manifest_content, + media_type="text/markdown", + headers={"Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff"}, + ) + + +async def public_hub_entries(request: Request, store: PluginUploadStore | None) -> tuple[AgentSkillsIndexEntry, ...]: + if store is None or marketplace_auth_required(): + return () + entries: Final[list[AgentSkillsIndexEntry]] = [] # mutable-ok: bounded inventory avoids copying on each append + async for item in store.public_metadata(): + entry: AgentSkillsIndexEntry | None = _index_entry(request, item) + if entry is None: + continue + entries.append(entry) + if len(entries) == 1000: + break + return tuple(entries) + + +def _index_entry(request: Request, item: PluginArchiveMetadata) -> AgentSkillsIndexEntry | None: + manifest: Final = MANIFEST_FIELDS.validate_json(item.manifest_json or "{}") + raw: Final = manifest.get("agent_skill") + if ( + not isinstance(raw, dict) + or not item.enabled + or not plugin_archive_source_matches( + request, + item.name, + item.source, + item.source.get("sha256", ""), + has_agent_skill_metadata=item.has_agent_skill_metadata, + ) + ): + return None + skill: Final = MANIFEST_FIELDS.validate_python(raw) + digest: Final = skill.get("digest") + description: Final = item.description or skill.get("description") + if not isinstance(digest, str) or not isinstance(description, str): + return None + return AgentSkillsIndexEntry( + name=item.name, + type="archive", + description=description, + url=f"{_download_url(request, item.name)}?digest={quote(digest, safe=':')}", + digest=digest, + ) diff --git a/litellm/proxy/discovery_endpoints/skill_hub_mutations.py b/litellm/proxy/discovery_endpoints/skill_hub_mutations.py new file mode 100644 index 00000000000..63c519923c6 --- /dev/null +++ b/litellm/proxy/discovery_endpoints/skill_hub_mutations.py @@ -0,0 +1,172 @@ +import hashlib +from dataclasses import dataclass +from datetime import datetime +from typing import Annotated, Final + +from fastapi import Depends, HTTPException +from typing_extensions import assert_never + +from litellm.models.plugin_upload import ( + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + PluginSnapshot, + PluginUploadStore, + PublishedPlugin, +) +from litellm.models.skill_submission import ( + SkillSubmission, + SkillSubmissionStore, + SubmissionConflict, + SubmissionLimitExceeded, + SubmissionMissing, + SubmissionReviewed, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import resolve_skill_visibility +from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin +from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 +from litellm.repositories.plugin_publication import resolve_plugin_version +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +@dataclass(frozen=True, slots=True) +class SkillMutationError: + status: int + message: str + + +def writer_error(caller: UserAPIKeyAuth) -> SkillMutationError | None: + if is_proxy_admin(caller): + return None + if ( + not caller.user_id + or caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID + or caller.user_role in (None, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY) + ): + return SkillMutationError(403, "Submitting a skill requires a writable user identity.") + return None + + +def submission_owner(caller: UserAPIKeyAuth) -> str | None | SkillMutationError: + if is_proxy_admin(caller) or caller.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + return None + if not caller.user_id or caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + return SkillMutationError(403, "Reading submissions requires a user identity.") + return caller.user_id + + +async def archive_writer(caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]) -> UserAPIKeyAuth: + from litellm.proxy.proxy_server import proxy_logging_obj + + error: Final = writer_error(caller) + if error is not None: + raise HTTPException(error.status, error.message) + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + raise HTTPException(503, "Skills Hub publishing requires the default v3 rate limiter") + key_identity: Final = caller.api_key or caller.token + identity: Final = ( + f"user:{caller.user_id}" if caller.user_id else f"key:{key_identity}" if key_identity else "local-admin" + ) + result: Final = await limiter.atomic_check_and_increment_by_n( + descriptors=[ + { + "key": "skill_hub_publish", + "value": hashlib.sha256(identity.encode()).hexdigest(), + "rate_limit": {"requests_per_unit": 10, "window_size": 60}, + } + ], + increments=[{"requests": 1}], + parent_otel_span=caller.parent_otel_span, + ) + if result["overall_code"] == "OVER_LIMIT": + raise HTTPException( + 429, "Skills Hub allows 10 uploads or imports per minute per user", headers={"Retry-After": "60"} + ) + return caller + + +async def write_target( + name: str, + updating: bool, + uploads: PluginUploadStore | None, + caller: UserAPIKeyAuth, + expected_updated_at: datetime | None = None, +) -> PluginSnapshot | SkillMutationError | None: + error: Final = writer_error(caller) + if error is not None: + return error + if uploads is None: + return SkillMutationError(503, "Skills Hub requires a connected database") + existing: Final = await uploads.get_plugin(name) + if updating and (existing is None or not (await resolve_skill_visibility(caller)).allows(existing)): + return SkillMutationError(404, "Skill not found") + if existing is not None and expected_updated_at is not None and existing.updated_at != expected_updated_at: + return SkillMutationError(409, "The published skill changed. Reload it before submitting.") + if not updating and existing is not None: + return SkillMutationError(409, f"A skill named '{name}' already exists. Update the existing skill instead.") + return existing + + +def publication_error(error: PluginPublicationError) -> SkillMutationError: + return SkillMutationError({"invalid_version": 400, "conflict": 409, "not_found": 404}[error.kind], error.message) + + +async def write_skill( + publication: PluginPublication, + existing: PluginSnapshot | None, + uploads: PluginUploadStore, + submissions: SkillSubmissionStore | None, + caller: UserAPIKeyAuth, +) -> PublishedPlugin | SkillSubmission | SkillMutationError: + error: Final = writer_error(caller) + if error is not None: + return error + validation: Final = resolve_plugin_version( + publication.version, None if existing is None else existing.version, updating=existing is not None + ) + if isinstance(validation, PluginPublicationError): + return publication_error(validation) + expected: Final = None if existing is None else PluginPublicationBase(existing.id, existing.updated_at) + if is_proxy_admin(caller): + published: Final = await uploads.publish(publication, expected) + return publication_error(published) if isinstance(published, PluginPublicationError) else published + if submissions is None or not caller.user_id: + return SkillMutationError(503, "Skills Hub requires a connected database") + submission: Final = await submissions.submit(publication, caller.user_id, expected) + return ( + SkillMutationError(429, submission.message) if isinstance(submission, SubmissionLimitExceeded) else submission + ) + + +async def review_skill( + submission_id: str, + submissions: SkillSubmissionStore | None, + caller: UserAPIKeyAuth, + approve: bool, + reason: str | None, + publication_base_url: str | None = None, +) -> SubmissionReviewed | SkillMutationError: + if not is_proxy_admin(caller): + return SkillMutationError(403, "Only proxy admins may review skill submissions.") + if submissions is None: + return SkillMutationError(503, "Skills Hub requires a connected database") + result: Final = await submissions.review( + submission_id, caller.user_id or "proxy-admin", approve, reason, publication_base_url=publication_base_url + ) + match result: + case SubmissionMissing(): + return SkillMutationError(404, "Skill submission not found") + case SubmissionConflict(message=message): + return SkillMutationError(409, message) + case PluginPublicationError(kind="invalid_version") as error: + return publication_error(error) + case PluginPublicationError(message=message): + return SkillMutationError(409, message) + case SubmissionReviewed(): + return result + case _: + assert_never(result) diff --git a/litellm/proxy/discovery_endpoints/skill_hub_submissions.py b/litellm/proxy/discovery_endpoints/skill_hub_submissions.py new file mode 100644 index 00000000000..c1bb422b2a0 --- /dev/null +++ b/litellm/proxy/discovery_endpoints/skill_hub_submissions.py @@ -0,0 +1,192 @@ +import asyncio +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from litellm.models.plugin_upload import PluginPublication +from litellm.models.skill_submission import SkillSubmission, SkillSubmissionStore, SubmissionStatus +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import resolve_skill_visibility +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.repositories import PluginUploadRepositoryDep, SkillSubmissionRepositoryDep +from litellm.proxy.discovery_endpoints.agent_skills_archive import build_agent_skill_archive +from litellm.proxy.discovery_endpoints.skill_hub_endpoints import ( + skill_metadata_response, + skill_proposal_response, + skill_publication_response, +) +from litellm.proxy.discovery_endpoints.skill_hub_mutations import SkillMutationError, review_skill, submission_owner +from litellm.types.proxy.skill_hub import ( + SkillHubRejection, + SkillHubReviewResponse, + SkillHubSubmission, + SkillHubSubmissionDetail, + SkillHubSubmissionList, +) + +router: Final = APIRouter(prefix="/skill-hub/submissions", tags=["Skills Hub"]) + + +def _store(submissions: SkillSubmissionStore | None) -> SkillSubmissionStore: + if submissions is None: + raise HTTPException(503, "Skills Hub requires a connected database") + return submissions + + +def _owner(caller: UserAPIKeyAuth) -> str | None: + owner: Final = submission_owner(caller) + if isinstance(owner, SkillMutationError): + raise HTTPException(owner.status, owner.message) + return owner + + +def _response(request: Request, submission: SkillSubmission) -> SkillHubSubmission: + return SkillHubSubmission( + id=submission.id, + skill_name=submission.plugin_name, + operation=submission.operation, + status=submission.status, + proposal=skill_proposal_response(request, submission), + has_archive=submission.has_archive, + submitted_by=submission.submitted_by, + created_at=submission.created_at, + base_skill_id=submission.base_plugin_id, + base_updated_at=submission.base_updated_at, + reviewed_by=submission.reviewed_by, + reviewed_at=submission.reviewed_at, + rejection_reason=submission.rejection_reason, + ) + + +@router.get("", response_model=SkillHubSubmissionList) +async def list_submissions( + request: Request, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + status: SubmissionStatus | None = None, + limit: Annotated[int, Query(ge=1, le=100)] = 20, + offset: Annotated[int, Query(ge=0)] = 0, +) -> SkillHubSubmissionList: + """Read a page of owned proposals. Proxy admins and admin viewers can read every owner's proposals.""" + owner: Final = _owner(caller) + page: Final = await _store(submissions).list(owner, status, limit, offset) + return SkillHubSubmissionList( + submissions=tuple(_response(request, row) for row in page.submissions), + count=len(page.submissions), + has_more=page.has_more, + ) + + +@router.get("/{submission_id}", response_model=SkillHubSubmissionDetail) +async def get_submission( + request: Request, + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubSubmissionDetail: + """Read an immutable proposal and its currently visible published skill.""" + owner: Final = _owner(caller) + submission: Final = await _store(submissions).get(submission_id, owner) + if submission is None: + raise HTTPException(404, "Skill submission not found") + if uploads is None: + raise HTTPException(503, "Skills Hub requires a connected database") + current: Final = await uploads.get_metadata(submission.plugin_name) + visible: Final = current is not None and (owner is None or (await resolve_skill_visibility(caller)).allows(current)) + return SkillHubSubmissionDetail( + submission=_response(request, submission), + current_skill=skill_metadata_response(request, current) if current is not None and visible else None, + ) + + +@router.get( + "/{submission_id}/archive", + response_class=Response, + responses={200: {"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}}}}, +) +async def download_submission( + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """Download an owned proposal as a private portable ZIP. Legacy nonportable archives return 404.""" + owner: Final = _owner(caller) + store: Final = _store(submissions) + submission: Final = await store.get(submission_id, owner) + if submission is None: + raise HTTPException(404, "Skill submission not found") + content: Final = await store.get_archive(submission_id, owner) + archive: Final = ( + await asyncio.to_thread(build_agent_skill_archive, content, plugin_name=submission.plugin_name) + if content is not None + else None + ) + if archive is None: + raise HTTPException(404, "This submission has no portable Agent Skills archive") + return Response( + archive.content, + media_type="application/zip", + headers={ + "Content-Disposition": f'attachment; filename="{submission.plugin_name}.zip"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "ETag": f'"{archive.digest}"', + }, + ) + + +async def _review( + request: Request, + submission_id: str, + submissions: SkillSubmissionStore | None, + caller: UserAPIKeyAuth, + approve: bool, + reason: str | None, +) -> SkillHubReviewResponse: + from litellm.proxy.utils import get_custom_url + + result: Final = await review_skill( + submission_id, submissions, caller, approve, reason, get_custom_url(str(request.base_url)) + ) + if isinstance(result, SkillMutationError): + raise HTTPException(result.status, result.message) + proposal: Final = result.submission.proposal + publication: Final = PluginPublication( + name=proposal.name, + version=proposal.version, + description=proposal.description, + manifest={ + **proposal.manifest, + **({"source": dict(result.published_source)} if result.published_source else {}), + }, + created_by=result.submission.submitted_by, + ) + return SkillHubReviewResponse( + action="approved" if approve else "rejected", + submission=_response(request, result.submission), + skill=None if result.plugin is None else skill_publication_response(request, publication, result.plugin), + ) + + +@router.put("/{submission_id}/approve", response_model=SkillHubReviewResponse) +async def approve_submission( + request: Request, + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubReviewResponse: + """Publish a pending proposal and record its approval atomically. Only proxy admins may review.""" + return await _review(request, submission_id, submissions, caller, True, None) + + +@router.put("/{submission_id}/reject", response_model=SkillHubReviewResponse) +async def reject_submission( + request: Request, + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + body: SkillHubRejection | None = None, +) -> SkillHubReviewResponse: + """Reject a pending proposal without changing the published skill. Only proxy admins may review.""" + return await _review(request, submission_id, submissions, caller, False, None if body is None else body.reason) diff --git a/litellm/proxy/discovery_endpoints/skill_import.py b/litellm/proxy/discovery_endpoints/skill_import.py new file mode 100644 index 00000000000..a74e3abef10 --- /dev/null +++ b/litellm/proxy/discovery_endpoints/skill_import.py @@ -0,0 +1,126 @@ +import io +import re +import zipfile +from dataclasses import dataclass +from typing import Final +from urllib.parse import quote, unquote, urlsplit + +import httpx + +from litellm.proxy.discovery_endpoints.agent_skills_archive import ( + MAX_ARCHIVE_ENTRIES, + MAX_ARCHIVE_UNPACKED_BYTES, + MAX_SKILL_MANIFEST_BYTES, + SKILL_ARCHIVE_ERRORS, + SkillArchive, + agent_skill_frontmatter, + build_agent_skill_archive, + read_skill_archive_entry, + safe_skill_archive_entry, + skill_archive_entries, +) + +MAX_IMPORT_BYTES: Final = 10 * 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class SkillImportError: + status: int + message: str + + +@dataclass(frozen=True, slots=True) +class GitHubSkillSource: + owner: str + repository: str + ref: str + skill: str | None + path: str | None + + @property + def archive_url(self) -> str: + return f"https://codeload.github.com/{self.owner}/{self.repository}/zip/{quote(self.ref, safe='')}" + + +def parse_skill_source( + url: str, *, skill: str | None, ref: str, path: str | None +) -> GitHubSkillSource | SkillImportError: + try: + parsed: Final = urlsplit(url) + except ValueError: + return SkillImportError(400, "Invalid skill source URL") + if parsed.scheme != "https" or parsed.netloc not in ("skills.sh", "github.com") or parsed.query or parsed.fragment: + return SkillImportError(400, "Use an HTTPS skills.sh skill URL or a public github.com repository URL") + parts: Final = tuple(unquote(part) for part in parsed.path.strip("/").split("/")) + if len(parts) < 2 or any(re.fullmatch(r"[A-Za-z0-9_.-]+", part) is None for part in parts[:2]): + return SkillImportError(400, "The source URL must identify a GitHub owner and repository") + if parsed.netloc == "skills.sh": + if len(parts) != 3 or (skill is not None and skill != parts[2]): + return SkillImportError(400, "Use skills.sh/// with a matching skill selection") + return GitHubSkillSource(parts[0], parts[1], ref, parts[2], path) + if len(parts) == 2: + return GitHubSkillSource(parts[0], parts[1].removesuffix(".git"), ref, skill, path) + if len(parts) < 5 or parts[2] != "tree" or path is not None or ref != "HEAD": + return SkillImportError(400, "Use a repository URL with ref and path, or a GitHub tree URL") + return GitHubSkillSource(parts[0], parts[1], parts[3], skill, "/".join(parts[4:])) + + +async def fetch_skill_source(source: GitHubSkillSource, client: httpx.AsyncClient) -> bytes | SkillImportError: + buffer: Final = io.BytesIO() + request: Final = httpx.Request( + "GET", + source.archive_url, + headers={"Accept": "application/zip"}, + extensions={"timeout": httpx.Timeout(30).as_dict()}, + ) + try: + response: Final = await client.send(request, stream=True, follow_redirects=False, auth=None) + try: + if response.status_code != 200: + return SkillImportError(422, "Unable to download the public GitHub repository at the requested ref") + async for chunk in response.aiter_bytes(): + if buffer.tell() + len(chunk) > MAX_IMPORT_BYTES: + return SkillImportError(413, "Repository ZIP must be 10 MiB or smaller") + buffer.write(chunk) + finally: + await response.aclose() + except httpx.HTTPError: + return SkillImportError(502, "The skill source could not be reached") + return buffer.getvalue() + + +def select_imported_skill(content: bytes, source: GitHubSkillSource) -> SkillArchive | SkillImportError: + try: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + infos: Final = archive.infolist() + if len(infos) > MAX_ARCHIVE_ENTRIES or sum(info.file_size for info in infos) > MAX_ARCHIVE_UNPACKED_BYTES: + return SkillImportError(413, "Repository exceeds the skill archive limits") + entries: Final = skill_archive_entries(archive, for_upload=True) + if entries is None: + return SkillImportError(422, "Repository contains unsafe archive entries") + candidates: Final = tuple(info for info, path in entries if path.endswith("/SKILL.md")) + if any(not safe_skill_archive_entry(info) for info in candidates): + return SkillImportError(422, "Repository contains unsafe skill manifests") + selected: Final = tuple(info for info in candidates if _matches_selection(info, archive, source)) + if len(selected) != 1: + return SkillImportError(422, "Select exactly one skill using its name or repository-relative path") + root: Final = selected[0].filename.rsplit("/", 1)[0] + skill: Final = build_agent_skill_archive(content, skill_path=root) + return ( + skill + if skill is not None + else SkillImportError(422, "The selected directory is not a valid Agent Skill") + ) + except SKILL_ARCHIVE_ERRORS: + return SkillImportError(422, "The repository did not return a readable skill ZIP") + + +def _matches_selection(info: zipfile.ZipInfo, archive: zipfile.ZipFile, source: GitHubSkillSource) -> bool: + if info.file_size > MAX_SKILL_MANIFEST_BYTES: + return False + relative: Final = info.filename.split("/", 1)[1].rpartition("/")[0] + if source.path is not None and relative != source.path.strip("/"): + return False + if source.skill is None: + return True + return agent_skill_frontmatter(read_skill_archive_entry(archive, info)).get("name") == source.skill diff --git a/litellm/proxy/discovery_endpoints/skill_scanning.py b/litellm/proxy/discovery_endpoints/skill_scanning.py new file mode 100644 index 00000000000..ee236482fbf --- /dev/null +++ b/litellm/proxy/discovery_endpoints/skill_scanning.py @@ -0,0 +1,186 @@ +import asyncio +import io +import json +from datetime import datetime, timedelta, timezone +from typing import Annotated, Final +from uuid import uuid4 + +import httpx +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException +from pydantic import ValidationError + +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory has untyped optional params +) +from litellm.models.skill_submission import SkillScanInput, SkillSubmissionStore, SubmissionConflict, SubmissionMissing +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin +from litellm.proxy.deps.repositories import SkillSubmissionRepositoryDep +from litellm.secret_managers.main import get_secret_str +from litellm.types.proxy.skill_scanning import ( + SkillScan, + SkillScannerConfig, + SkillScanReport, + SkillScanRequest, + SkillScanResponse, +) + +router: Final = APIRouter() +_MAX_REPORT_BYTES: Final = 1024 * 1024 + + +def scanner_config() -> SkillScannerConfig | str | None: + from litellm.proxy.proxy_server import general_settings_view + + value: Final = general_settings_view().get("skill_scanner") + if value is None: + return None + try: + return SkillScannerConfig.model_validate(value) + except ValidationError: + return "The skill scanner configuration is invalid" + + +async def scan_report( + client: httpx.AsyncClient, config: SkillScannerConfig, api_key: str, target: SkillScanInput +) -> SkillScanReport | str: + try: + async with client.stream( + "POST", + config.endpoint, + headers={"Authorization": f"Bearer {api_key}", "Accept-Encoding": "identity"}, + data={ + "source": json.dumps(dict(target.source)), + "use_llm": str(config.use_llm).lower(), + "expires_at": target.scan.expires_at.isoformat(), + }, + files=None if target.content is None else {"file": ("skill.zip", target.content, "application/zip")}, + timeout=httpx.Timeout(config.timeout_seconds, connect=10), + follow_redirects=False, + ) as response: + if response.status_code != 200: + return f"Scanner returned HTTP {response.status_code}. Check the scanner service and authentication." + output: Final = io.BytesIO() + async for chunk in response.aiter_bytes(chunk_size=65536): + if output.tell() + len(chunk) > _MAX_REPORT_BYTES: + return "Scanner report exceeded the 1 MiB limit." + output.write(chunk) + report: Final = SkillScanReport.model_validate_json(output.getvalue()) + if config.use_llm and report.scan_mode != "static+llm": + return report.model_copy( + update={ + "complete": False, + "limitations": (*report.limitations[:99], "Requested semantic analysis did not run."), + } + ) + return report + except (httpx.HTTPError, ValidationError): + return "The scanner was unreachable or returned an invalid report." + + +async def complete_scan( + submissions: SkillSubmissionStore, + submission_id: str, + target: SkillScanInput, + config: SkillScannerConfig, + api_key: str, +) -> None: + try: + client: Final = get_async_httpx_client( + "skill_scanner", params={"ssl_verify": True, "follow_redirects": False} + ).client + report: Final = await asyncio.wait_for( + scan_report(client, config, api_key, target), + timeout=(target.scan.expires_at - datetime.now(timezone.utc)).total_seconds(), + ) + completed: Final = target.scan.model_copy( + update={ + "status": "completed" if isinstance(report, SkillScanReport) else "failed", + "report": report if isinstance(report, SkillScanReport) else None, + "error": report if isinstance(report, str) else None, + "completed_at": datetime.now(timezone.utc), + } + ) + await submissions.finish_scan(submission_id, completed) + except (TimeoutError, asyncio.CancelledError) as error: + await submissions.finish_scan( + submission_id, + target.scan.model_copy( + update={ + "status": "failed", + "error": "Scan timed out or was interrupted. Try scanning again.", + "completed_at": datetime.now(timezone.utc), + } + ), + ) + if isinstance(error, asyncio.CancelledError): + raise + + +@router.get( + "/skill-hub/submissions/{submission_id}/scan", + tags=["Skills Hub"], + response_model=SkillScanResponse, +) +async def get_skill_scan( + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillScanResponse: + """Read the latest manual security scan. Only proxy admins may read reports.""" + if not is_proxy_admin(caller): + raise HTTPException(403, "Proxy admin access is required to review security scans") + if submissions is None: + raise HTTPException(503, "Database is not connected") + scan: Final = await submissions.get_scan(submission_id) + if isinstance(scan, SubmissionMissing): + raise HTTPException(404, "Skill submission not found") + return SkillScanResponse( + configured=isinstance(scanner_config(), SkillScannerConfig), + scan=None if scan is None else scan.at(datetime.now(timezone.utc)), + ) + + +@router.post( + "/skill-hub/submissions/{submission_id}/scan", + tags=["Skills Hub"], + response_model=SkillScanResponse, +) +async def start_skill_scan( + submission_id: str, + background_tasks: BackgroundTasks, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + _request: Annotated[SkillScanRequest | None, Body()] = None, +) -> SkillScanResponse: + """Start an optional scan of a pending submission. Scans never change approval decisions.""" + if not is_proxy_admin(caller): + raise HTTPException(403, "Proxy admin access is required to scan skills") + if submissions is None: + raise HTTPException(503, "Database is not connected") + config: Final = scanner_config() + if config is None: + raise HTTPException(503, "Security scanning is not configured") + if isinstance(config, str): + raise HTTPException(503, config) + api_key: Final = get_secret_str(config.api_key_env) + if not api_key: + raise HTTPException(503, "The scanner authentication secret is not configured") + now: Final = datetime.now(timezone.utc) + target: Final = await submissions.start_scan( + submission_id, + SkillScan( + id=str(uuid4()), + status="running", + requested_by=caller.user_id or "proxy-admin", + started_at=now, + expires_at=now + timedelta(seconds=config.timeout_seconds), + ), + ) + if isinstance(target, SubmissionMissing): + raise HTTPException(404, "Skill submission not found") + if isinstance(target, SubmissionConflict): + raise HTTPException(409, target.message) + background_tasks.add_task(complete_scan, submissions, submission_id, target, config, api_key) + return SkillScanResponse(configured=True, scan=target.scan) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 1ddbc139564..55e2e129e36 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from functools import partial from itertools import groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import billed_guard from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES from litellm.proxy.db.spend_log_batching import spend_log_write_batches from litellm.proxy.utils import PrismaClient +from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ( DailyGuardrailMetricsRepository, DailyGuardrailUsageUnitsRepository, @@ -40,7 +41,7 @@ _RowKey = TypeVar("_RowKey") _RowValue = TypeVar("_RowValue") -class _UsageUnitKey(NamedTuple): +class UsageUnitKey(NamedTuple): guardrail_id: str date: str team_id: str @@ -48,7 +49,7 @@ class _UsageUnitKey(NamedTuple): usage_unit: str -class _UsageUnitIncrement(NamedTuple): +class UsageUnitIncrement(NamedTuple): units: int cost: float """USD for the priced share of units.""" @@ -56,14 +57,14 @@ class _UsageUnitIncrement(NamedTuple): """Units recorded with no known price, the share cost leaves out.""" -def _usage_unit_key(entry: tuple[_UsageUnitKey, _UsageUnitIncrement]) -> _UsageUnitKey: +def _usage_unit_key(entry: tuple[UsageUnitKey, UsageUnitIncrement]) -> UsageUnitKey: return entry[0] -def _usage_unit_increment(units: int, cost: float | None) -> _UsageUnitIncrement: +def _usage_unit_increment(units: int, cost: float | None) -> UsageUnitIncrement: if cost is None: - return _UsageUnitIncrement(units=units, cost=0.0, untracked_units=units) - return _UsageUnitIncrement(units=units, cost=cost, untracked_units=0) + return UsageUnitIncrement(units=units, cost=0.0, untracked_units=units) + return UsageUnitIncrement(units=units, cost=cost, untracked_units=0) class _MetricsKey(NamedTuple): @@ -89,22 +90,22 @@ class PendingRollups: def __init__(self) -> None: self.lock: Final = asyncio.Lock() self.metrics: Mapping[_MetricsKey, Mapping[str, int]] = MappingProxyType({}) - self.units: Mapping[_UsageUnitKey, _UsageUnitIncrement] = MappingProxyType({}) + self.units: Mapping[UsageUnitKey, UsageUnitIncrement] = MappingProxyType({}) _PENDING_ROLLUPS: Final = PendingRollups() _NO_COUNTERS: Final[Mapping[str, int]] = MappingProxyType({}) -_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0, untracked_units=0) +_NO_INCREMENT: Final = UsageUnitIncrement(units=0, cost=0.0, untracked_units=0) def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object]) -> tuple[_RowKey, ...]: return (*base, *(key for key in extra if key not in base)) -def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitIncrement: +def _summed_increments(increments: Iterable[UsageUnitIncrement]) -> UsageUnitIncrement: materialized: Final = tuple(increments) - return _UsageUnitIncrement( + return UsageUnitIncrement( units=sum(i.units for i in materialized), cost=sum(i.cost for i in materialized), untracked_units=sum(i.untracked_units for i in materialized), @@ -112,8 +113,8 @@ def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitI def _merged_unit_rows( - base: Mapping[_UsageUnitKey, _UsageUnitIncrement], extra: Mapping[_UsageUnitKey, _UsageUnitIncrement] -) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: + base: Mapping[UsageUnitKey, UsageUnitIncrement], extra: Mapping[UsageUnitKey, UsageUnitIncrement] +) -> Mapping[UsageUnitKey, UsageUnitIncrement]: return MappingProxyType( { key: _summed_increments((base.get(key, _NO_INCREMENT), extra.get(key, _NO_INCREMENT))) @@ -250,7 +251,7 @@ def _parse_payload_start_time(payload: Mapping[str, object]) -> datetime | None: def _iter_usage_unit_increments( logs_to_process: Sequence[Mapping[str, object]], -) -> Iterator[tuple[_UsageUnitKey, _UsageUnitIncrement]]: +) -> Iterator[tuple[UsageUnitKey, UsageUnitIncrement]]: for payload in logs_to_process: start_time = _parse_payload_start_time(payload) if not payload.get("request_id") or start_time is None: @@ -266,14 +267,14 @@ def _iter_usage_unit_increments( cost_by_unit = billed_guardrail_cost_by_unit(entry) for unit_name, units in usage.items(): if isinstance(units, int) and not isinstance(units, bool) and units > 0: - key = _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)) + key = UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)) cost = cost_by_unit.get(str(unit_name)) if cost_by_unit is not None else None yield key, _usage_unit_increment(units=units, cost=cost) def _sum_usage_unit_increments( logs_to_process: Sequence[Mapping[str, object]], -) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: +) -> Mapping[UsageUnitKey, UsageUnitIncrement]: ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=_usage_unit_key) return MappingProxyType( { @@ -283,8 +284,16 @@ def _sum_usage_unit_increments( ) -async def _upsert_usage_unit_row( - prisma_client: PrismaClient, key: _UsageUnitKey, increment: _UsageUnitIncrement +class GuardrailUsageDatabase(Protocol): + @property + def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... + + +async def upsert_usage_unit_row( + prisma_client: PrismaClient, + key: UsageUnitKey, + increment: UsageUnitIncrement, + outer_transaction: GuardrailUsageDatabase | None = None, ) -> None: row: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsCreateInput] = { "guardrail_id": key.guardrail_id, @@ -314,7 +323,12 @@ async def _upsert_usage_unit_row( "untracked_units": {"increment": increment.untracked_units}, }, } - await DailyGuardrailUsageUnitsRepository(prisma_client).table.upsert(where=where, data=data) + table: Final = ( + outer_transaction.litellm_dailyguardrailusageunits + if outer_transaction is not None + else DailyGuardrailUsageUnitsRepository(prisma_client).table + ) + await table.upsert(where=where, data=data) async def _upsert_metrics_row(prisma_client: PrismaClient, key: _MetricsKey, agg: Mapping[str, int]) -> None: @@ -363,7 +377,7 @@ async def process_spend_logs_guardrail_usage( ) index_rows_by_key: Final[dict[tuple[str, str], dict[str, object]]] = {} - for payload in logs_to_process: + for payload in (row for row in logs_to_process if row.get("accounting_log_disabled") is not True): request_id = payload.get("request_id") start_time = _parse_payload_start_time(payload) if not isinstance(request_id, str) or not request_id or start_time is None: @@ -416,7 +430,10 @@ async def process_spend_logs_guardrail_usage( {key: agg for key, agg in daily_guardrail.items() if int(agg["requests_evaluated"]) > 0} ) metrics_rows: Final = _merged_metric_rows(pending_metrics, evaluated_metrics) - unit_rows: Final = _merged_unit_rows(pending_units, _sum_usage_unit_increments(logs_to_process)) + unit_rows: Final = _merged_unit_rows( + pending_units, + _sum_usage_unit_increments(tuple(row for row in logs_to_process if "accounting_payload" not in row)), + ) if not metrics_rows and not index_rows and not unit_rows: return @@ -435,7 +452,7 @@ async def process_spend_logs_guardrail_usage( metrics_rows, partial(_upsert_metrics_row, prisma_client), "daily metrics", sleep ) failed_units: Final = await _upsert_rows_with_retry( - unit_rows, partial(_upsert_usage_unit_row, prisma_client), "usage unit", sleep + unit_rows, partial(upsert_usage_unit_row, prisma_client), "usage unit", sleep ) if failed_metrics or failed_units: async with pending.lock: diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 90887422567..ff67f2cb7c6 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -9,6 +9,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast from pydantic import TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -38,6 +39,7 @@ from litellm.proxy.auth.auth_checks import ( from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.db_spend_update_writer import ( + ACCOUNTING_LEDGER_KWARG, DBSpendUpdateWriter, debitable_model_access_groups, get_llm_router, @@ -657,6 +659,7 @@ class _ProxyDBLogger(CustomLogger): token: Final = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) end_user_id: Final = None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id + accounting_rows: Final = tuple((cost, dict(row)) for cost, row in spend_rows) charged: Final = [ await self._spend_writer().update_database( token=token, @@ -664,17 +667,17 @@ class _ProxyDBLogger(CustomLogger): user_id=user_api_key_dict.user_id, end_user_id=end_user_id, team_id=user_api_key_dict.team_id, - kwargs=dict(row_kwargs), + kwargs=row_kwargs, completion_response=original_exception, start_time=start_time, end_time=datetime.now(), org_id=user_api_key_dict.org_id, project_id=user_api_key_dict.project_id, ) - for row_cost, row_kwargs in spend_rows + for row_cost, row_kwargs in accounting_rows ] counted_rows: Final = tuple( - row for row, row_charged in zip(spend_rows, charged, strict=True) if row_charged and row[0] > 0 + row for row, row_charged in zip(accounting_rows, charged, strict=True) if row_charged and row[0] > 0 ) if counted_rows and not ProxyUpdateSpend.disable_spend_updates(): await self._increment_failure_spend_counters( @@ -753,6 +756,7 @@ class _ProxyDBLogger(CustomLogger): router=get_llm_router(), ), project_id=user_api_key_dict.project_id, + **_ledger_counter_options(row_kwargs.get(ACCOUNTING_LEDGER_KWARG)), ) except Exception: # noqa: BLE001 # one row's counter failure must not keep the other rows' counters from landing verbose_proxy_logger.exception("Failed to raise spend counters for a failure spend row") @@ -1196,6 +1200,14 @@ def _get_request_tags_for_cost_tracking( return None +class _LedgerCounterOptions(TypedDict): + windows_from_ledger: ReadOnly[NotRequired[bool]] + + +def _ledger_counter_options(ledger: object) -> _LedgerCounterOptions: + return {"windows_from_ledger": True} if ledger is True else {} + + class _ChargeSpend(Protocol): async def __call__( self, @@ -1256,6 +1268,7 @@ class _IncrementSpendCounters(Protocol): request_started_at: datetime | None = None, model_access_groups: Sequence[str] | None = None, project_id: str | None = None, + windows_from_ledger: bool = False, ) -> None: ... @@ -1393,6 +1406,7 @@ async def _update_database_and_spend_counters_in_batch( request_started_at=start_time, model_access_groups=model_access_groups, project_id=project_id, + **_ledger_counter_options(_BATCH_CALLBACK_FIELDS.validate_python(kwargs).get(ACCOUNTING_LEDGER_KWARG)), ) except Exception: if budget_reservation is not None: diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 6e4ea0bd0cb..bbd6a1c9021 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -287,12 +287,12 @@ def compute_tag_metadata_totals(records: Sequence[DailySpendRecord]) -> SpendMet def _entity_metadata( - entity_metadata_field: Mapping[str, dict[str, object]] | None, + entity_metadata_field: Mapping[str, Mapping[str, object]] | None, entity_id: str, ) -> dict[str, object]: """The metadata payload for one entity breakdown bucket, empty when the caller passed none.""" stored: Final = entity_metadata_field.get(entity_id) if entity_metadata_field else None - return stored if stored is not None else {} + return dict(stored) if stored is not None else {} def provider_bucket_metadata(provider: str, base: Mapping[str, object]) -> Mapping[str, object]: @@ -1039,7 +1039,7 @@ def _fold_entity_rollups_sync( results: Sequence[DailySpendData], entity_rows: Sequence[EntityRollupRow], api_key_metadata: Mapping[str, KeyMetadataRow], - entity_metadata_field: Mapping[str, dict[str, object]] | None, # mutable-ok: shared field shape + entity_metadata_field: Mapping[str, Mapping[str, object]] | None, ) -> None: """Write breakdown.entities onto the already-built per-day results.""" by_date: Final = {day.date.strftime("%Y-%m-%d"): day for day in results} @@ -1113,6 +1113,19 @@ async def _guardrail_spend_or_empty( return () +async def _user_entity_metadata( + repository: DailyActivityRepository, entity_rows: Sequence[EntityRollupRow] +) -> Mapping[str, Mapping[str, object]]: + user_ids: Final = frozenset(row.entity_id for row in entity_rows if row.entity_id) + users: Final = await repository.user_metadata(user_ids) + return MappingProxyType( + { + user_id: MappingProxyType({"user_email": user.user_email, "user_alias": user.user_alias}) + for user_id, user in users.items() + } + ) + + async def get_daily_activity_aggregated( repository: DailyActivityRepository, scope: DailyActivityScope, @@ -1157,12 +1170,17 @@ async def get_daily_activity_aggregated( if entity_api_keys else MappingProxyType({}) ) + user_entity_metadata: Final[Mapping[str, Mapping[str, object]]] = ( + await _user_entity_metadata(repository, entity_records) + if scope.table is DailyActivityTable.USER + else MappingProxyType({}) + ) await asyncio.to_thread( _fold_entity_rollups_sync, results=aggregated["results"], entity_rows=entity_records, api_key_metadata=entity_key_metadata, - entity_metadata_field=entity_metadata_field, + entity_metadata_field=MappingProxyType({**user_entity_metadata, **(entity_metadata_field or {})}), ) results: Final = ( list(await asyncio.to_thread(_with_guardrail_breakdown, aggregated["results"], guardrail_rows)) diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index 7ff73ac6e2e..1eb6ab3210b 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -363,7 +363,7 @@ USER_RESOLVER = EntityScopeResolver( tags=("Budget & Spend Tracking", "Internal User management"), query=_user_query, resolve=_resolve_user, - include_entity_breakdown=False, + include_entity_breakdown=True, include_guardrail_breakdown=True, operation_names=MappingProxyType( { diff --git a/litellm/proxy/management_helpers/team_roster_sync.py b/litellm/proxy/management_helpers/team_roster_sync.py index e10d13bf030..6090d68304c 100644 --- a/litellm/proxy/management_helpers/team_roster_sync.py +++ b/litellm/proxy/management_helpers/team_roster_sync.py @@ -384,7 +384,22 @@ async def sync_team_roster( user_api_key_dict=user_api_key_dict, litellm_changed_by=None, ) - await _settle_caches(synced, sorted(to_add | to_remove), removal, user_api_key_cache, proxy_logging_obj) + cleanup: Final = asyncio.create_task( + _settle_caches(synced, sorted(to_add | to_remove), removal, user_api_key_cache, proxy_logging_obj) + ) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as cancellation: + # The transaction committed. Finish revocation before propagating cancellation. + try: + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + cleanup.result() + finally: + raise cancellation _emit_team_members_metric(synced) _schedule_team_membership_audit_log( team_id=team_id, diff --git a/litellm/proxy/middleware/request_size_limit_middleware.py b/litellm/proxy/middleware/request_size_limit_middleware.py index f2883168d65..329a3c38d91 100644 --- a/litellm/proxy/middleware/request_size_limit_middleware.py +++ b/litellm/proxy/middleware/request_size_limit_middleware.py @@ -5,6 +5,7 @@ from typing import Final from fastapi import HTTPException from starlette._utils import get_route_path +from starlette.routing import compile_path from starlette.types import ASGIApp, Message, Receive, Scope, Send from litellm.proxy.common_utils.anthropic_error_payload import route_error_content @@ -25,7 +26,20 @@ def effective_max_request_size_mb( ) -> float | None: global_limit: Final = configured if configured is not None and configured > 0 else None method: Final = scope.get("method") - route_limit: Final = route_limits.get((method, get_route_path(scope))) if isinstance(method, str) else None + path: Final = get_route_path(scope) + exact_limit: Final = route_limits.get((method, path)) if isinstance(method, str) else None + route_limit: Final = ( + exact_limit + if exact_limit is not None + else next( + ( + limit + for (route_method, route), limit in route_limits.items() + if route_method == method and "{" in route and compile_path(route)[0].fullmatch(path) is not None + ), + None, + ) + ) if route_limit is None: return global_limit return min(route_limit, global_limit) if global_limit is not None else route_limit diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 61d8e2d7bb5..e23238bd64b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6,6 +6,7 @@ import importlib import inspect import io import itertools +import logging import os import random import re @@ -423,9 +424,13 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( + router as model_deprecation_alerts_router, +) from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) +from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload from litellm.proxy.auth.auth_checks import ( ROLE_BASED_PERMISSIONS_ADAPTER, ExperimentalUIJWTToken, @@ -2937,7 +2942,11 @@ def configured_max_request_size_mb() -> int | float | None: _ROUTE_MAX_REQUEST_SIZE_MB: Final[Mapping[tuple[str, str], float]] = MappingProxyType( - {("POST", "/claude-code/plugins/upload"): 11} + { + ("POST", "/claude-code/plugins/upload"): 11, + ("POST", "/skill-hub/skills/upload"): 11, + ("PUT", "/skill-hub/skills/{name}/upload"): 11, + } ) @@ -3624,6 +3633,7 @@ async def increment_spend_counters( request_started_at: datetime | None = None, model_access_groups: Sequence[str] | None = None, project_id: str | None = None, + windows_from_ledger: bool = False, ): """ Atomically increment spend counters for budget enforcement. @@ -3660,6 +3670,7 @@ async def increment_spend_counters( request_started_at=request_started_at, model_access_groups=model_access_groups, project_id=project_id, + windows_from_ledger=windows_from_ledger, ) @@ -3689,6 +3700,50 @@ def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bo user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key) +async def accounting_window_keys(token: str | None, team_id: str | None) -> tuple[Mapping[str, object], ...]: + scopes: Final = tuple( + (entity_type, entity_id, cache_key) + for entity_type, entity_id, cache_key in ( + (Litellm_EntityType.KEY, token, token), + (Litellm_EntityType.TEAM, team_id, f"team_id:{team_id}"), + ) + if entity_id is not None and cache_key is not None + ) + snapshots: Final = tuple( + [ + await _accounting_scope_windows(entity_type, entity_id, cache_key) + for entity_type, entity_id, cache_key in scopes + ] + ) + from itertools import chain + + return tuple(chain.from_iterable(snapshots)) + + +async def _accounting_scope_windows( + entity_type: Litellm_EntityType, entity_id: str, cache_key: str +) -> tuple[Mapping[str, object], ...]: + from litellm.proxy.spend_tracking.budget_reservation import fixed_budget_window + + cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) + limits: Final = ( + await _stored_budget_limits(entity_type, entity_id) if cached is None else _cached_budget_limits(cached) + ) + if not isinstance(limits, list): + return () + windows: Final = TypeAdapter(tuple[object, ...]).validate_python(limits) + return tuple( + { + "entity_type": entity_type.value, + "entity_id": entity_id, + "window_duration": fixed[0], + "window_start": fixed[1].isoformat(), + } + for window in windows + if (fixed := fixed_budget_window(window)) is not None + ) + + async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object: """Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache.""" if prisma_client is None: @@ -3730,6 +3785,7 @@ async def _increment_spend_counters_batched( request_started_at: datetime | None, model_access_groups: Sequence[str] | None, project_id: str | None = None, + windows_from_ledger: bool = False, ): """Runs inside one spend counter batch: the reservation reconcile and the warm checks share a single MGET, and the reconcile adjustments go out in the same INCRBYFLOAT pipeline as the counter increments.""" @@ -3771,15 +3827,16 @@ async def _increment_spend_counters_batched( if window_counter not in reserved_counter_keys else None ) - await _enqueue_window_spend_row_update( - entity_type=entity_type, - entity_id=entity_id, - reset_at=window_reset_at, - window_duration=duration, - window_start=window_start, - increment=cost, - request_started_at=request_started_at, - ) + if not windows_from_ledger: + await _enqueue_window_spend_row_update( + entity_type=entity_type, + entity_id=entity_id, + reset_at=window_reset_at, + window_duration=duration, + window_start=window_start, + increment=cost, + request_started_at=request_started_at, + ) return pending_window async def _window_increments( @@ -13400,6 +13457,13 @@ async def completion( _response.choices[0].text = e.message return _response except Exception as e: + if is_provider_overload(e): + raise await base_llm_response_processor.handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, @@ -18631,7 +18695,11 @@ async def model_deprecations( -H "Authorization: Bearer $LITELLM_MASTER_KEY" ``` """ - return collect_model_deprecations(llm_router=llm_router, warn_within_days=warn_within_days) + return collect_model_deprecations( + llm_router=llm_router, + warn_within_days=warn_within_days, + successor_log_level=logging.DEBUG, + ) def get_model_group_info( @@ -22398,6 +22466,7 @@ app.include_router(prompt_caching_requests_router) app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) +app.include_router(model_deprecation_alerts_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index c007b047087..818f2dc4858 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1616,10 +1616,69 @@ model LiteLLM_ClaudeCodePluginTable { created_by String? uploaded_archive LiteLLM_ClaudeCodePluginUpload? + versions LiteLLM_ClaudeCodePluginVersion[] + downloads LiteLLM_ClaudeCodePluginDownload[] + anonymous_download_count BigInt @default(0) @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginVersion { + id String @id @default(uuid()) + plugin_id String + version String? + description String? + manifest_json String? + published_at DateTime? + published_by String? + archive_available Boolean @default(false) + content Bytes? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@unique([plugin_id, version]) + @@index([plugin_id, published_at]) +} + + +model LiteLLM_SkillSubmission { + id String @id @default(uuid()) + plugin_name String + operation String + status String @default("pending_review") + proposal_json String + archive_content Bytes? + submitted_by String + created_at DateTime @default(now()) + base_plugin_id String? + base_updated_at DateTime? + reviewed_by String? + reviewed_at DateTime? + rejection_reason String? + scan_json String? + + @@index([submitted_by, created_at(sort: Desc), id(sort: Desc)]) + @@index([submitted_by, status, created_at(sort: Desc), id(sort: Desc)]) + @@index([status, created_at(sort: Desc), id(sort: Desc)]) + @@index([created_at(sort: Desc), id(sort: Desc)]) +} + +model LiteLLM_ClaudeCodePluginDownload { + id BigInt @id @default(autoincrement()) + plugin_id String + downloaded_at DateTime @default(now()) + version String? + digest String + authenticated Boolean + user_id String? + user_email String? + key_alias String? + team_id String? + team_alias String? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@index([plugin_id, id]) +} + model LiteLLM_ClaudeCodePluginUpload { plugin_name String @id content Bytes @@ -2071,74 +2130,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index dcfd6148f54..af7eb742aa7 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1264,6 +1264,15 @@ def _get_entry_reserved_cost(entry: dict, default_reserved_cost: float) -> float return default_reserved_cost +def fixed_budget_window(window: object) -> tuple[str, datetime] | None: + fields: Final = _coerce_window(window) + start: Final = get_budget_window_start(window) + duration: Final = fields.get("budget_duration") + if not fields.get("reset_at") or not isinstance(duration, str) or start is None: + return None + return duration, start + + def get_budget_window_start(window: object) -> datetime | None: window_dict: Final = _coerce_window(window) budget_duration: Final = window_dict.get("budget_duration") diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 85a7f870eb4..fc54d1da0b2 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -161,9 +161,7 @@ def _is_non_secret_key_value(value: str, *, key_alias: object = None) -> bool: ) -def _redact_logged_api_key( - value: str | None, *, already_redacted: bool = False, key_alias: object = None -) -> str | None: +def redact_logged_api_key(value: str | None, *, already_redacted: bool = False, key_alias: object = None) -> str | None: if not isinstance(value, str) or not value: return None stripped: Final = re.sub(r"(?i)^bearer ", "", value) @@ -327,7 +325,7 @@ def _get_spend_logs_metadata( and _is_non_secret_key_value(_trusted_hash, key_alias=_key_alias) and _trusted_hash == _raw_key ) - clean_metadata["user_api_key"] = _redact_logged_api_key( + clean_metadata["user_api_key"] = redact_logged_api_key( _raw_key, already_redacted=_already_redacted, key_alias=_key_alias ) clean_metadata["applied_guardrails"] = applied_guardrails @@ -692,14 +690,14 @@ def get_logging_payload( and _is_non_secret_key_value(_trusted_hash, key_alias=_key_alias) and _trusted_hash == api_key ) - api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted, key_alias=_key_alias) or "" + api_key = redact_logged_api_key(api_key, already_redacted=_key_already_redacted, key_alias=_key_alias) or "" if ( standard_logging_payload is not None ): # [TODO] migrate completely to sl payload. currently missing pass-through endpoint data api_key = ( api_key - or _redact_logged_api_key( + or redact_logged_api_key( standard_logging_payload["metadata"].get("user_api_key_hash"), already_redacted=True, key_alias=standard_logging_payload["metadata"].get("user_api_key_alias"), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 145c8e38229..a2928bba287 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -28,6 +28,7 @@ from dataclasses import dataclass, field from datetime import date, datetime, timedelta, timezone from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from email.utils import formatdate from functools import partial, reduce from itertools import takewhile from operator import attrgetter @@ -213,6 +214,7 @@ from litellm.proxy.db.db_url_settings import ( from litellm.proxy.db.exception_handler import ( PrismaDBExceptionHandler, call_with_db_reconnect_retry, + db_write_never_sent, ) from litellm.proxy.db.health_check_latest import ( LatestHealthCheckRow, @@ -277,6 +279,7 @@ from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.spend_log_insert import RejectedSpendLogRow from litellm.repositories.table_repositories import ( EndUserRepository, HealthCheckRepository, @@ -1950,6 +1953,7 @@ class ProxyLogging: self.daily_report_started: bool = False self.hanging_requests_check_started: bool = False self.deprecation_check_started: bool = False + self.deprecation_email_check_started: bool = False def startup_event( self, @@ -1993,6 +1997,20 @@ class ProxyLogging: self.hanging_requests_check_started = True self._ensure_deprecation_check_scheduled() + self._ensure_deprecation_email_check_scheduled() + + def _ensure_deprecation_email_check_scheduled(self) -> None: + """The email event opt-in controls reminders independently of Slack integration settings.""" + if self.deprecation_email_check_started: + return + try: + asyncio.get_running_loop() + except RuntimeError: + return + from litellm.integrations.model_deprecation_email import run_scheduled_model_deprecation_emails + + asyncio.create_task(run_scheduled_model_deprecation_emails()) + self.deprecation_email_check_started = True def _ensure_deprecation_check_scheduled(self) -> None: """Alerting can be configured at startup or by a later config reload, so schedule from either path""" @@ -2039,6 +2057,7 @@ class ProxyLogging: if updated_slack_alerting is True: self._ensure_deprecation_check_scheduled() + self._ensure_deprecation_email_check_scheduled() self.slack_alerting_instance.update_values( alerting=self.alerting, alerting_threshold=self.alerting_threshold, @@ -8261,7 +8280,9 @@ async def send_email( receiver_email: str | None = None, subject: str | None = None, html: str | None = None, -): + text: str | None = None, + message_id: str | None = None, +) -> bool: """ smtp_host, smtp_port, @@ -8287,16 +8308,21 @@ async def send_email( raise ValueError(f"No HTML body provided for SMTP email. {html}") ## EMAIL SETUP ## - email_message: Final = MIMEMultipart() + email_message: Final = MIMEMultipart("alternative" if text is not None else "mixed") email_message["From"] = sender_email email_message["To"] = receiver_email email_message["Subject"] = subject + if message_id is not None: + email_message["Message-ID"] = message_id + email_message["Date"] = formatdate(usegmt=True) verbose_proxy_logger.debug("sending email from %s to %s", sender_email, receiver_email) if smtp_host is None: raise ValueError("Trying to use SMTP, but SMTP_HOST is not set") # Attach the body to the email + if text is not None: + email_message.attach(MIMEText(text, "plain")) email_message.attach(MIMEText(html, "html")) try: @@ -8315,6 +8341,8 @@ async def send_email( except Exception as e: verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e)) + return False + return True def hash_token(token: str): @@ -8417,14 +8445,17 @@ async def enqueue_spend_logs( """ added: Final = sum(spend_log_row_bytes(row) for row in logs) async with prisma_client._spend_log_transactions_lock: - queued: Final = ( + queued: Final[tuple[Mapping[str, object], ...]] = ( tuple(logs) + tuple(prisma_client.spend_log_transactions) if at_head else tuple(prisma_client.spend_log_transactions) + tuple(logs) ) kept, kept_bytes = spend_log_queue_within_budget(queued, PrismaClient.spend_log_queue_bytes + added, max_bytes) - prisma_client.spend_log_transactions[:] = kept - PrismaClient.spend_log_queue_bytes = kept_bytes + rescued: Final = tuple( + accounting_only_spend_log(row) for row in queued[: len(queued) - len(kept)] if _carries_a_charge(row) + ) + prisma_client.spend_log_transactions[:] = (*rescued, *kept) + PrismaClient.spend_log_queue_bytes = kept_bytes + sum(spend_log_row_bytes(row) for row in rescued) if len(kept) < len(queued): verbose_proxy_logger.error( "Spend tracking - spend log queue is at its %d byte budget; dropped the %d oldest spend logs", @@ -8479,13 +8510,18 @@ class ProxyUpdateSpend: prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, end_user_list_transactions: dict[str, float], + outer_transaction: _EndUserSpendTransaction | None = None, ): for i in range(n_retry_times + 1): start_time = time.time() try: async with ( db_span("update_end_user_spend", "LiteLLM_EndUserTable"), - _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) as transaction, + ( + contextlib.nullcontext(outer_transaction) + if outer_transaction is not None + else _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) + ) as transaction, ): batcher: _EndUserSpendBatch async with transaction.batch_() as batcher: @@ -8507,6 +8543,8 @@ class ProxyUpdateSpend: break except Exception as e: + if outer_transaction is not None: + raise await DBSpendUpdateWriter.handle_spend_update_failure( e=e, attempt=i, @@ -8546,11 +8584,18 @@ class ProxyUpdateSpend: for i in range(n_retry_times + 1): try: base_url = os.getenv("SPEND_LOGS_URL", None) - if len(logs_to_process) > 0 and base_url is not None and db_writer_client is not None: + await _commit_export_accounting( + prisma_client, rows_to_write, base_url if db_writer_client is not None else None + ) + if ( + base_url is not None + and db_writer_client is not None + and any(row.get("accounting_log_disabled") is not True for row in rows_to_write) + ): if not base_url.endswith("/"): base_url += "/" verbose_proxy_logger.debug("base_url: %s", base_url) - json_data = json.dumps(rows_to_write) + json_data = json.dumps(_spend_log_export_rows(rows_to_write)) response = await db_writer_client.post( url=base_url + "spend/update", data=json_data, @@ -8628,6 +8673,28 @@ class ProxyUpdateSpend: return False +def _spend_log_export_rows(rows: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple( + { + key: value + for key, value in row.items() + if key not in {"accounting_payload", "accounting_log_disabled", "accounting_delivery_id"} + } + for row in rows + if row.get("accounting_log_disabled") is not True + ) + + +async def _commit_export_accounting( + prisma_client: PrismaClient, rows: Sequence[Mapping[str, object]], base_url: str | None +) -> None: + if base_url is None: + return + await SpendLogsRepository(prisma_client).insert_with_deliveries( + tuple(row for row in rows if "accounting_payload" in row), write_spend_logs=False + ) + + async def commit_tool_discoveries(prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging) -> None: """Scheduler job writing discovered tools to LiteLLM_ToolTable, on its own lock, next to update_spend.""" writer: Final = proxy_logging_obj.db_spend_update_writer @@ -8803,9 +8870,20 @@ async def update_daily_tag_spend( def _rollup_batch_never_applied(e: Exception) -> bool: """True when a drained rollup batch provably never reached its rows: Postgres had no - connection for it (53300) or cancelled it under ``lock_timeout`` before it took the row - lock (55P03). Either way re-sending the batch cannot double-count, so it is requeued.""" - return PrismaDBExceptionHandler.is_database_capacity_error(e) or PrismaDBExceptionHandler.is_lock_timeout_error(e) + connection for it (53300), was unreachable or refusing connections (P1001, 57P03), or + cancelled it under ``lock_timeout`` before it took the row lock (55P03). In each case + re-sending the batch cannot double-count, so it is requeued.""" + return ( + PrismaDBExceptionHandler.is_database_capacity_error(e) + or db_write_never_sent(e) + or PrismaDBExceptionHandler.is_lock_timeout_error(e) + ) + + +def _rollup_failure_stops_job(e: Exception) -> bool: + """True when the database cannot take any write, so the job stops instead of draining + the queues after this one into the same failure.""" + return PrismaDBExceptionHandler.is_database_capacity_error(e) or db_write_never_sent(e) async def update_spend_logs_job( @@ -8896,12 +8974,12 @@ async def _run_spend_logs_job( *prisma_client.tool_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections or rollup row lock timed out during tool usage " + "Spend tracking - database unavailable or rollup row lock timed out during tool usage " "flush, requeued %s tool usage transactions for the next flush: %s", len(tool_usage_to_process), tool_tracking_err, ) - if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err): + if _rollup_failure_stops_job(tool_tracking_err): raise else: verbose_proxy_logger.error( @@ -8928,12 +9006,12 @@ async def _run_spend_logs_job( *prisma_client.model_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections or rollup row lock timed out during model usage " + "Spend tracking - database unavailable or rollup row lock timed out during model usage " "flush, requeued %s model usage transactions for the next flush: %s", len(model_usage_to_process), model_usage_err, ) - if PrismaDBExceptionHandler.is_database_capacity_error(model_usage_err): + if _rollup_failure_stops_job(model_usage_err): raise else: verbose_proxy_logger.error( @@ -9110,10 +9188,45 @@ def _is_transient_spend_log_write_error(e: Exception) -> bool: return ( PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) + or PrismaDBExceptionHandler.is_database_lifecycle_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e) ) +_ACCOUNTING_ONLY_KEYS: Final = frozenset({"request_id", "call_type", "accounting_payload", "accounting_delivery_id"}) + + +def _carries_a_charge(row: Mapping[str, object]) -> bool: + return "accounting_payload" in row + + +def accounting_only_spend_log(row: Mapping[str, object]) -> Mapping[str, object]: + return { + **{key: value for key, value in row.items() if key in _ACCOUNTING_ONLY_KEYS}, + "accounting_log_disabled": True, + } + + +async def _keep_charges_of_dropped_logs(repo: SpendLogsRepository, rows: Sequence[Mapping[str, object]]) -> None: + charged: Final = tuple(accounting_only_spend_log(row) for row in rows if _carries_a_charge(row)) + if not charged: + return + try: + await repo.insert_with_deliveries(charged) + except Exception as e: # noqa: BLE001 # the request log is already lost; keep the flush going for other rows + if not ( + PrismaDBExceptionHandler.is_rejected_row_error(e) + or isinstance(e, (UnicodeEncodeError, RejectedSpendLogRow)) + ): + raise + spend_log_error( + "Spend tracking - could not keep the charges of %d dropped spend log rows. error=%s", + len(charged), + str(e), + exc=e, + ) + + async def _create_spend_logs_with_poison_isolation( repo: SpendLogsRepository, rows: Sequence[Mapping[str, object]], @@ -9142,10 +9255,13 @@ async def _create_spend_logs_with_poison_isolation( persist. Returns the budget left after this subtree. """ try: - await repo.table.create_many(data=rows, skip_duplicates=True) + await repo.insert_with_deliveries(rows) return failure_budget except Exception as e: - if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)): + if not ( + PrismaDBExceptionHandler.is_rejected_row_error(e) + or isinstance(e, (UnicodeEncodeError, RejectedSpendLogRow)) + ): raise if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): raise @@ -9160,6 +9276,7 @@ async def _create_spend_logs_with_poison_isolation( str(e), exc=e, ) + await _keep_charges_of_dropped_logs(repo, rows) return budget_left if budget_left <= 0: spend_log_error( @@ -9167,6 +9284,7 @@ async def _create_spend_logs_with_poison_isolation( "isolation failure budget exhausted for this flush", len(rows), ) + await _keep_charges_of_dropped_logs(repo, rows) return 0 mid: Final = len(rows) // 2 remaining: Final = await _create_spend_logs_with_poison_isolation(repo, rows[:mid], budget_left) @@ -9176,6 +9294,7 @@ async def _create_spend_logs_with_poison_isolation( "isolation failure budget exhausted for this flush", len(rows) - mid, ) + await _keep_charges_of_dropped_logs(repo, rows[mid:]) return 0 return await _create_spend_logs_with_poison_isolation(repo, rows[mid:], remaining) diff --git a/litellm/repositories/accounting_delivery_repository.py b/litellm/repositories/accounting_delivery_repository.py new file mode 100644 index 00000000000..7dd92aecb2d --- /dev/null +++ b/litellm/repositories/accounting_delivery_repository.py @@ -0,0 +1,193 @@ +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import timedelta +from itertools import chain +from types import TracebackType +from typing import TYPE_CHECKING, Final, Literal, Protocol + +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm.repositories.prisma_protocols import TableActions + +if TYPE_CHECKING: + from litellm.proxy.db.db_spend_update_writer import SpendBatchManager + + +class LedgerTransaction(Protocol): + @property + def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + async def execute_raw(self, query: str, *args: object) -> int: ... + + def batch_(self) -> "SpendBatchManager": ... + + +class _TransactionManager(Protocol): + async def __aenter__(self) -> LedgerTransaction: ... + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + + +class LedgerDatabase(Protocol): + def tx(self, *, timeout: timedelta) -> _TransactionManager: ... + + +class _Delivery(BaseModel): + model_config = ConfigDict(frozen=True) + + delivery_id: str + attempts: int + accounting_payload: JsonValue + + +@dataclass(frozen=True, slots=True) +class DeliveryProjection: + row: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class DeliveryBatchResult: + applied: int = 0 + failed: int = 0 + parked: tuple[str, ...] = () + error: str | None = None + + +class DeliveryMetrics(BaseModel): + model_config = ConfigDict(frozen=True) + + pending_count: int + parked_count: int + oldest_pending_seconds: float + batch_count: int + table_bytes: int + + +_DELIVERIES: Final = TypeAdapter(tuple[_Delivery, ...]) +_PAYLOAD: Final = TypeAdapter(dict[str, object]) +_TIMEOUT: Final = timedelta(seconds=60) +_CLAIM: Final = """SELECT delivery_id, attempts, accounting_payload FROM "LiteLLM_SpendLogDelivery" +WHERE state = 'pending' AND next_attempt_at <= CURRENT_TIMESTAMP + AND (attempts > 0) = $2::boolean +ORDER BY next_attempt_at, delivery_id LIMIT $1::integer FOR UPDATE SKIP LOCKED""" +_APPLIED: Final = """UPDATE "LiteLLM_SpendLogDelivery" +SET state = 'applied', applied_at = CURRENT_TIMESTAMP, last_error = NULL +WHERE delivery_id = ANY($1::text[]) AND state = 'pending' """ +_FAILED: Final = """UPDATE "LiteLLM_SpendLogDelivery" +SET attempts = attempts + 1, last_error = $3, + next_attempt_at = CURRENT_TIMESTAMP + make_interval(secs => LEAST(300, power(2, LEAST(attempts, 8)))::integer), + state = $4::"AccountingDeliveryState" +WHERE delivery_id = $1 AND state = 'pending' AND attempts = $2::integer +RETURNING delivery_id, attempts, accounting_payload""" + + +def _is_permanent_failure(error: Exception) -> bool: + """A delivery whose own data failed fails the same way on every retry, while any other error is the database's + and clears once it recovers.""" + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler + + if PrismaDBExceptionHandler.is_database_service_unavailable_error(error): + return False + return isinstance(error, (ValueError, TypeError, KeyError)) or PrismaDBExceptionHandler.is_rejected_row_error(error) + + +class AccountingDeliveryRepository: + def __init__(self, database: LedgerDatabase) -> None: + self._database = database + + async def metrics(self) -> DeliveryMetrics: + async with self._database.tx(timeout=_TIMEOUT) as transaction: + rows: Final = await transaction.query_raw( + "SELECT COUNT(*) FILTER (WHERE state = 'pending') AS pending_count, " + "COUNT(*) FILTER (WHERE state = 'parked') AS parked_count, " + "COALESCE(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - MIN(created_at) FILTER (WHERE state = 'pending')), 0) " + "AS oldest_pending_seconds, COUNT(*) FILTER (WHERE is_batch) AS batch_count, " + "pg_total_relation_size('\"LiteLLM_SpendLogDelivery\"') AS table_bytes " + 'FROM "LiteLLM_SpendLogDelivery"' + ) + return DeliveryMetrics.model_validate(rows[0]) + + async def cleanup_applied(self, *, retention_days: int = 1, batch_size: int = 1000) -> int: + if retention_days < 1 or batch_size < 1: + raise ValueError("Delivery retention and cleanup batch size must be positive") + async with self._database.tx(timeout=_TIMEOUT) as transaction: + return await transaction.execute_raw( + 'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id IN (' + 'SELECT delivery_id FROM "LiteLLM_SpendLogDelivery" ' + "WHERE state = 'applied' AND NOT is_batch AND applied_at < CURRENT_TIMESTAMP - make_interval(days => $1::integer) " + "ORDER BY applied_at, delivery_id LIMIT $2::integer FOR UPDATE SKIP LOCKED)", + retention_days, + batch_size, + ) + + async def apply_pending( + self, + apply: Callable[[LedgerTransaction, tuple[DeliveryProjection, ...]], Awaitable[None]], + *, + batch_size: int = 100, + max_attempts: int = 5, + retry_only: bool | None = None, + ) -> DeliveryBatchResult: + if batch_size < 1 or max_attempts < 1: + raise ValueError("Delivery batch size and maximum attempts must be positive") + claimed: tuple[_Delivery, ...] = () # rebind-ok: preserve claims for retry accounting after rollback + try: + async with self._database.tx(timeout=_TIMEOUT) as transaction: + retries: Final = ( + _DELIVERIES.validate_python(await transaction.query_raw(_CLAIM, 1, True)) + if retry_only is not False + else () + ) + claimed = retries or ( # rebind-ok: retain claimed identities after rollback + _DELIVERIES.validate_python(await transaction.query_raw(_CLAIM, batch_size, False)) + if retry_only is not True + else () + ) + if not claimed: + return DeliveryBatchResult() + await apply( + transaction, + tuple( + DeliveryProjection(_PAYLOAD.validate_python(delivery.accounting_payload)) + for delivery in claimed + ), + ) + await transaction.execute_raw(_APPLIED, tuple(delivery.delivery_id for delivery in claimed)) + return DeliveryBatchResult(applied=len(claimed)) + except Exception as error: # noqa: BLE001 # roll back all destinations before recording a retry + if not claimed: + raise + return await self._record_failure(claimed, error, max_attempts) + + async def _record_failure( + self, claimed: tuple[_Delivery, ...], error: Exception, max_attempts: int + ) -> DeliveryBatchResult: + parks: Final = len(claimed) == 1 and _is_permanent_failure(error) + async with self._database.tx(timeout=_TIMEOUT) as transaction: + updates: Final = tuple( + [ + await self._record_row_failure( + transaction, + delivery, + type(error).__name__, + "parked" if parks and delivery.attempts + 1 >= max_attempts else "pending", + ) + for delivery in claimed + ] + ) + return DeliveryBatchResult( + failed=len(claimed), parked=tuple(chain.from_iterable(updates)), error=type(error).__name__ + ) + + @staticmethod + async def _record_row_failure( + transaction: LedgerTransaction, delivery: _Delivery, error: str, state: Literal["pending", "parked"] + ) -> tuple[str, ...]: + updated: Final = _DELIVERIES.validate_python( + await transaction.query_raw(_FAILED, delivery.delivery_id, delivery.attempts, error, state) + ) + return tuple(row.delivery_id for row in updated) if state == "parked" else () diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index 8768d79b0dd..36c6ab7f711 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -4,6 +4,7 @@ from __future__ import annotations import json from collections.abc import Mapping, Sequence +from datetime import datetime from typing import TYPE_CHECKING, Final, Protocol, cast if TYPE_CHECKING: @@ -35,6 +36,8 @@ class _ConfigTable(Protocol): async def delete(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ... + async def delete_many(self, *, where: Mapping[str, object]) -> int: ... + class _ConfigDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> object: ... @@ -130,3 +133,9 @@ class ConfigRepository: param_value = _decoded_json(param_value) result[record.param_name] = param_value return result + + async def delete_params_older_than(self, prefix: str, before: datetime) -> int: + """Expire operational config records using the timestamp written by revision claims.""" + return await self._config_table.delete_many( + where={"param_name": {"startswith": prefix}, "last_run_at": {"lt": before}} + ) diff --git a/litellm/repositories/plugin_publication.py b/litellm/repositories/plugin_publication.py new file mode 100644 index 00000000000..f5fab458c6b --- /dev/null +++ b/litellm/repositories/plugin_publication.py @@ -0,0 +1,175 @@ +import hashlib +import json +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Final, Protocol + +from pydantic import TypeAdapter +from semver import Version + +from litellm.models.plugin_upload import ( + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + PublishedPlugin, +) +from litellm.repositories.prisma_protocols import TableActions + +if TYPE_CHECKING: + from prisma.models import ( + LiteLLM_ClaudeCodePluginTable, + LiteLLM_ClaudeCodePluginUpload, + LiteLLM_ClaudeCodePluginVersion, + ) + +_MANIFEST: Final = TypeAdapter(dict[str, object]) +_SOURCE: Final = TypeAdapter(dict[str, str]) + + +class PluginPublicationTransaction(Protocol): + @property + def litellm_claudecodeplugintable(self) -> TableActions["LiteLLM_ClaudeCodePluginTable"]: ... + + @property + def litellm_claudecodepluginupload(self) -> TableActions["LiteLLM_ClaudeCodePluginUpload"]: ... + + @property + def litellm_claudecodepluginversion(self) -> TableActions["LiteLLM_ClaudeCodePluginVersion"]: ... + + async def query_raw(self, query: str, *args: object) -> object: ... + + +def resolve_plugin_version( + requested: str | None, current: str | None = None, *, updating: bool = False +) -> str | PluginPublicationError: + try: + previous: Final = Version.parse(current) if current is not None else None + except ValueError: + return _legacy_version(requested) + if updating and previous is None: + return _legacy_version(requested) + try: + candidate: Final = Version.parse(requested) if requested is not None else None + except ValueError: + return PluginPublicationError("invalid_version", "Version must be a valid semantic version, for example 1.0.1") + if candidate is None: + return str(previous.bump_patch()) if previous is not None else "1.0.0" + if previous is not None and candidate.compare(previous) <= 0: + return PluginPublicationError("invalid_version", f"Version must be strictly higher than {current}") + return str(candidate) + + +def _legacy_version(requested: str | None) -> str | PluginPublicationError: + if requested is None: + return PluginPublicationError( + "conflict", + "The published version is missing or is not SemVer. Supply an explicit semantic version to update it", + ) + return resolve_plugin_version(requested) + + +def verified_archive(content: bytes | None, manifest_json: str | None) -> bytes | None: + if content is None: + return None + manifest: Final = _MANIFEST.validate_json(manifest_json or "{}") + source: Final = _SOURCE.validate_python(manifest.get("source", {})) + if source.get("source") != "archive" or source.get("sha256", "").lower() != hashlib.sha256(content).hexdigest(): + return None + return content + + +async def _snapshot( + tx: PluginPublicationTransaction, + plugin: "LiteLLM_ClaudeCodePluginTable", + content: bytes | None, + published_by: str | None, +) -> None: + from prisma import Base64 + + archive: Final = verified_archive(content, plugin.manifest_json) + await tx.litellm_claudecodepluginversion.create( + data={ + "plugin_id": plugin.id, + "version": plugin.version, + "description": plugin.description, + "manifest_json": plugin.manifest_json, + "published_at": plugin.updated_at, + "published_by": published_by, + "content": Base64.encode(archive) if archive is not None else None, + "archive_available": archive is not None, + } + ) + + +async def publish_plugin( + tx: PluginPublicationTransaction, + publication: PluginPublication, + expected: PluginPublicationBase | None = None, +) -> PublishedPlugin | PluginPublicationError: + from prisma import Base64 + + await tx.query_raw('SELECT id FROM "LiteLLM_ClaudeCodePluginTable" WHERE name = $1 FOR UPDATE', publication.name) + current: Final = await tx.litellm_claudecodeplugintable.find_unique(where={"name": publication.name}) + if current is not None and expected is None: + return PluginPublicationError("conflict", f"A skill named '{publication.name}' already exists") + if expected is not None and current is None: + return PluginPublicationError("not_found", f"Plugin '{publication.name}' not found") + if current is not None and ( + expected is not None and (current.id != expected.id or current.updated_at != expected.updated_at) + ): + return PluginPublicationError("conflict", "The published skill changed. Reload it before publishing an update") + version: Final = resolve_plugin_version( + publication.version, current.version if current is not None else None, updating=expected is not None + ) + if isinstance(version, PluginPublicationError): + return version + manifest_json: Final = json.dumps({**publication.manifest, "version": version}) + if publication.content is not None and verified_archive(publication.content, manifest_json) is None: + return PluginPublicationError("conflict", "Uploaded archive does not match its published source digest") + timestamp: Final = datetime.now(timezone.utc) + data: Final = { + "version": version, + "description": publication.description, + "manifest_json": manifest_json, + "files_json": "{}", + "updated_at": timestamp, + } + if current is None: + created: Final = await tx.litellm_claudecodeplugintable.create( + data={**data, "name": publication.name, "created_by": publication.created_by} + ) + if publication.content is not None: + await tx.litellm_claudecodepluginupload.create( + data={"plugin_name": publication.name, "content": Base64.encode(publication.content)} + ) + await _snapshot(tx, created, publication.content, publication.created_by) + return PublishedPlugin(created.id, created.name, version, created.enabled, created.updated_at or timestamp) + previous_archive: Final = await tx.litellm_claudecodepluginupload.find_unique(where={"plugin_name": current.name}) + previous_content: Final = ( + verified_archive(previous_archive.content.decode(), current.manifest_json) + if previous_archive is not None + else None + ) + history_count: Final = await tx.litellm_claudecodepluginversion.count(where={"plugin_id": current.id}, take=1) + if history_count == 0: + await _snapshot(tx, current, previous_content, None) + unchanged_content: Final = ( + previous_content + if _MANIFEST.validate_json(current.manifest_json or "{}").get("source") == publication.manifest.get("source") + else None + ) + content: Final = publication.content if publication.content is not None else unchanged_content + updated: Final = await tx.litellm_claudecodeplugintable.update(where={"id": current.id}, data=data) + if updated is None: + raise RuntimeError("Locked plugin disappeared during publication") + if content is None: + await tx.litellm_claudecodepluginupload.delete_many(where={"plugin_name": current.name}) + else: + await tx.litellm_claudecodepluginupload.upsert( + where={"plugin_name": current.name}, + data={ + "create": {"plugin_name": current.name, "content": Base64.encode(content)}, + "update": {"content": Base64.encode(content)}, + }, + ) + await _snapshot(tx, updated, content, publication.created_by) + return PublishedPlugin(updated.id, updated.name, version, updated.enabled, updated.updated_at or timestamp) diff --git a/litellm/repositories/plugin_upload_repository.py b/litellm/repositories/plugin_upload_repository.py index 99f1f76abb5..99e226e442b 100644 --- a/litellm/repositories/plugin_upload_repository.py +++ b/litellm/repositories/plugin_upload_repository.py @@ -1,60 +1,197 @@ -import json +from collections.abc import AsyncIterator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma dynamically exposes its typed table actions +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # Prisma dynamically exposes its typed table actions +) -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError -from litellm.models.plugin_upload import PluginArchiveMetadata, PluginUpload +from litellm.models.plugin_upload import ( + PluginArchiveMetadata, + PluginDownloader, + PluginDownloadEvent, + PluginDownloadResult, + PluginDownloadsResponse, + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + PluginSnapshot, + PluginUpload, + PluginVersion, + PublishedPlugin, + has_agent_skill_metadata, +) +from litellm.repositories.plugin_publication import PluginPublicationTransaction, publish_plugin, verified_archive from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ClaudeCodePluginRepository if TYPE_CHECKING: from prisma import models as prisma_models + from litellm.proxy.db.create_views import SupportsExecuteRaw from litellm.proxy.utils import PrismaClient _MANIFEST: Final = TypeAdapter(dict[str, object]) _SOURCE: Final = TypeAdapter(dict[str, str]) +PAGE_SIZE: Final = 100 +_INCREMENT_ANONYMOUS_DOWNLOADS: Final = """ +UPDATE "LiteLLM_ClaudeCodePluginTable" +SET "anonymous_download_count" = "anonymous_download_count" + 1 +WHERE "id" = $1 +""" +_DOWNLOAD_COUNTS_QUERY: Final = """ +SELECT p.id, p.anonymous_download_count + COUNT(d.id) AS download_count +FROM "LiteLLM_ClaudeCodePluginTable" p +LEFT JOIN "LiteLLM_ClaudeCodePluginDownload" d ON d.plugin_id = p.id +WHERE p.id = ANY($1::text[]) +GROUP BY p.id +""" + + +@dataclass(frozen=True, slots=True) +class _DownloadCount: + id: str + download_count: int + + +_DOWNLOAD_COUNTS: Final = TypeAdapter(tuple[_DownloadCount, ...]) +_VERSIONS: Final = TypeAdapter(tuple[PluginVersion, ...]) +_VERSION_QUERY: Final = """ + SELECT v.id, v.version, v.description, COALESCE(v.manifest_json, '{}')::jsonb AS manifest, + v.published_at, v.published_by, v.archive_available + FROM "LiteLLM_ClaudeCodePluginVersion" v JOIN "LiteLLM_ClaudeCodePluginTable" p ON p.id = v.plugin_id + WHERE p.id = $1 AND p.enabled = $2 +""" + + +class _PublicationDatabase(PluginPublicationTransaction, Protocol): + def tx(self) -> AbstractAsyncContextManager[PluginPublicationTransaction]: ... class PluginUploadRepository: def __init__(self, prisma_client: "PrismaClient") -> None: + self._db = cast( # cast-ok: the generated Prisma transaction delegate is untyped at this boundary + _PublicationDatabase, prisma_client.writer_db + ) + self._writer = cast( # cast-ok: PrismaWrapper dynamically delegates raw statements to its generated client + "SupportsExecuteRaw", prisma_client.writer_db + ) self._plugins = ClaudeCodePluginRepository(prisma_client, use_writer=True).table self._archives = cast( # cast-ok: Prisma exposes generated table actions through an untyped delegate TableActions["prisma_models.LiteLLM_ClaudeCodePluginUpload"], prisma_client.writer_db.litellm_claudecodepluginupload, ) + self._downloads = cast( # cast-ok: Prisma exposes generated table actions through an untyped delegate + TableActions["prisma_models.LiteLLM_ClaudeCodePluginDownload"], + prisma_client.writer_db.litellm_claudecodeplugindownload, + ) async def create(self, upload: PluginUpload) -> str | None: - from prisma import Base64 + result: Final = await self.publish( + PluginPublication( + name=upload.name, + version=upload.version, + description=upload.description, + manifest=upload.manifest, + content=upload.content, + created_by=upload.created_by, + ) + ) + return result.id if isinstance(result, PublishedPlugin) else None + + async def publish( + self, publication: PluginPublication, expected: PluginPublicationBase | None = None + ) -> PublishedPlugin | PluginPublicationError: from prisma.errors import UniqueViolationError try: - plugin: Final = await self._plugins.create( - data={ - "name": upload.name, - "version": upload.version, - "description": upload.description, - "manifest_json": json.dumps(dict(upload.manifest)), - "created_by": upload.created_by, - "uploaded_archive": {"create": {"content": Base64.encode(upload.content)}}, - } - ) + async with self._db.tx() as tx: + return await publish_plugin(tx, publication, expected) except UniqueViolationError: + return PluginPublicationError("conflict", "A skill with this name or version already exists") + + async def versions(self, metadata: PluginArchiveMetadata, limit: int, offset: int) -> tuple[PluginVersion, ...]: + rows: Final = await self._db.query_raw( + _VERSION_QUERY + " ORDER BY v.published_at DESC NULLS LAST, v.id DESC LIMIT $3 OFFSET $4", + metadata.id, + metadata.enabled, + limit, + offset, + ) + return _VERSIONS.validate_python(rows) + + async def version(self, metadata: PluginArchiveMetadata, version_id: str) -> PluginVersion | None: + rows: Final = await self._db.query_raw( + _VERSION_QUERY + " AND v.id = $3", + metadata.id, + metadata.enabled, + version_id, + ) + versions: Final = _VERSIONS.validate_python(rows) + return versions[0] if versions else None + + async def version_content(self, metadata: PluginArchiveMetadata, version_id: str) -> bytes | None: + version: Final = await self._db.litellm_claudecodepluginversion.find_first( + where={ + "id": version_id, + "plugin_id": metadata.id, + "plugin": { + "is": { + "enabled": metadata.enabled, + } + }, + } + ) + if version is None or version.content is None: return None - return plugin.id + return verified_archive(version.content.decode(), version.manifest_json) + + async def get_plugin(self, name: str) -> PluginSnapshot | None: + row: Final = await self._plugins.find_unique(where={"name": name}) + if row is None: + return None + return PluginSnapshot( + id=row.id, + name=row.name, + version=row.version, + description=row.description, + manifest=MappingProxyType(_MANIFEST.validate_json(row.manifest_json or "{}")), + enabled=row.enabled, + created_at=row.created_at, + updated_at=row.updated_at, + created_by=row.created_by, + ) async def get_metadata(self, name: str) -> PluginArchiveMetadata | None: plugin: Final = await self._plugins.find_unique(where={"name": name}) if plugin is None: return None - manifest: Final = _MANIFEST.validate_json(plugin.manifest_json or "{}") - return PluginArchiveMetadata( - name=plugin.name, - enabled=plugin.enabled, - source=MappingProxyType(_SOURCE.validate_python(manifest.get("source", {}))), - manifest_json=plugin.manifest_json, - ) + return _archive_metadata(plugin) + + async def public_metadata(self) -> AsyncIterator[PluginArchiveMetadata]: + after: str | None = None # rebind-ok: bounded keyset traversal advances after each raw page + while True: + plugins: Sequence[prisma_models.LiteLLM_ClaudeCodePluginTable] = await self._plugins.find_many( + where={"enabled": True} if after is None else {"enabled": True, "name": {"gt": after}}, + take=PAGE_SIZE, + order={"name": "asc"}, + ) + if not plugins: + return + for plugin in plugins: + try: + metadata: PluginArchiveMetadata = _archive_metadata(plugin) + except ValidationError: + continue + yield metadata + if len(plugins) < PAGE_SIZE: + return + after = plugins[-1].name async def get_content(self, metadata: PluginArchiveMetadata) -> bytes | None: archive: Final = await self._archives.find_first( @@ -62,6 +199,7 @@ class PluginUploadRepository: "plugin_name": metadata.name, "plugin": { "is": { + "id": metadata.id, "manifest_json": metadata.manifest_json, "enabled": metadata.enabled, } @@ -69,3 +207,84 @@ class PluginUploadRepository: } ) return None if archive is None else archive.content.decode() + + async def record_download( + self, metadata: PluginArchiveMetadata, digest: str, downloader: PluginDownloader + ) -> PluginDownloadResult: + from prisma.errors import ForeignKeyViolationError + + if metadata.id is None: + return PluginDownloadResult.SKILL_NOT_FOUND + if not downloader.authenticated: + # A counter bounds anonymous storage; raw SQL preserves Prisma's @updatedAt metadata. + affected: Final = await self._writer.execute_raw(_INCREMENT_ANONYMOUS_DOWNLOADS, metadata.id) + return PluginDownloadResult.RECORDED if affected == 1 else PluginDownloadResult.SKILL_NOT_FOUND + try: + await self._downloads.create( + data={ + "plugin_id": metadata.id, + "version": metadata.version, + "digest": digest, + "authenticated": downloader.authenticated, + "user_id": downloader.user_id, + "user_email": downloader.user_email, + "key_alias": downloader.key_alias, + "team_id": downloader.team_id, + "team_alias": downloader.team_alias, + } + ) + except ForeignKeyViolationError: + return PluginDownloadResult.SKILL_NOT_FOUND + return PluginDownloadResult.RECORDED + + async def downloads(self, name: str, limit: int, cursor: int | None) -> PluginDownloadsResponse | None: + plugin: Final = await self._plugins.find_unique(where={"name": name}) + if plugin is None: + return None + total: Final = await self._downloads.count(where={"plugin_id": plugin.id}) + rows: Final = await self._downloads.find_many( + where={"plugin_id": plugin.id, **({"id": {"lt": cursor}} if cursor is not None else {})}, + order={"id": "desc"}, + take=limit + 1, + ) + return PluginDownloadsResponse( + total_downloads=total + plugin.anonymous_download_count, + anonymous_downloads=plugin.anonymous_download_count, + downloads=tuple( + PluginDownloadEvent( + id=str(row.id), + downloaded_at=row.downloaded_at, + version=row.version, + digest=row.digest, + authenticated=row.authenticated, + user_id=row.user_id, + user_email=row.user_email, + key_alias=row.key_alias, + team_id=row.team_id, + team_alias=row.team_alias, + ) + for row in rows[:limit] + ), + next_cursor=str(rows[limit - 1].id) if len(rows) > limit else None, + ) + + async def download_counts(self, plugin_ids: tuple[str, ...]) -> Mapping[str, int]: + if not plugin_ids: + return MappingProxyType({}) + rows: Final = _DOWNLOAD_COUNTS.validate_python(await self._db.query_raw(_DOWNLOAD_COUNTS_QUERY, plugin_ids)) + return MappingProxyType({row.id: row.download_count for row in rows}) + + +def _archive_metadata(plugin: "prisma_models.LiteLLM_ClaudeCodePluginTable") -> PluginArchiveMetadata: + manifest: Final = _MANIFEST.validate_json(plugin.manifest_json or "{}") + return PluginArchiveMetadata( + name=plugin.name, + enabled=plugin.enabled, + source=MappingProxyType(_SOURCE.validate_python(manifest.get("source", {}))), + manifest_json=plugin.manifest_json, + id=plugin.id, + version=plugin.version, + description=plugin.description, + has_agent_skill_metadata=has_agent_skill_metadata(manifest), + updated_at=plugin.updated_at, + ) diff --git a/litellm/repositories/skill_submission_repository.py b/litellm/repositories/skill_submission_repository.py new file mode 100644 index 00000000000..829aa0aadbf --- /dev/null +++ b/litellm/repositories/skill_submission_repository.py @@ -0,0 +1,368 @@ +import hashlib +import json +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager +from dataclasses import replace +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma's generated delegates are dynamic +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.models.plugin_upload import ( + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + PublishedPlugin, + uploaded_plugin_source, +) +from litellm.models.skill_submission import ( + SkillProposal, + SkillScanInput, + SkillSubmission, + SubmissionConflict, + SubmissionLimitExceeded, + SubmissionMissing, + SubmissionOperation, + SubmissionPage, + SubmissionReviewed, + SubmissionReviewResult, + SubmissionStatus, +) +from litellm.repositories.plugin_publication import PluginPublicationTransaction, publish_plugin, verified_archive +from litellm.repositories.prisma_protocols import TableActions +from litellm.types.proxy.skill_scanning import SkillScan + +if TYPE_CHECKING: + from prisma import models as prisma_models + + from litellm.proxy.utils import PrismaClient + + +class _StoredProposal(BaseModel): + model_config = ConfigDict(frozen=True) + + manifest: Mapping[str, object] + version: str | None + + def proposal(self, name: str) -> SkillProposal: + return SkillProposal( + name, self.version, _DESCRIPTION.validate_python(self.manifest.get("description")), self.manifest + ) + + +class _PendingUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + count: int + storage_bytes: int + + +class _SubmissionRow(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + plugin_name: str + operation: SubmissionOperation + status: SubmissionStatus + proposal_json: str + has_archive: bool + submitted_by: str + created_at: datetime + base_plugin_id: str | None + base_updated_at: datetime | None + reviewed_by: str | None + reviewed_at: datetime | None + rejection_reason: str | None + + def submission(self) -> SkillSubmission: + stored: Final = _StoredProposal.model_validate_json(self.proposal_json) + return SkillSubmission( + id=self.id, + plugin_name=self.plugin_name, + operation=self.operation, + status=self.status, + proposal=stored.proposal(self.plugin_name), + has_archive=self.has_archive, + submitted_by=self.submitted_by, + created_at=self.created_at, + base_plugin_id=self.base_plugin_id, + base_updated_at=self.base_updated_at, + reviewed_by=self.reviewed_by, + reviewed_at=self.reviewed_at, + rejection_reason=self.rejection_reason, + ) + + +_SCAN_ROWS: Final = TypeAdapter(tuple[Mapping[str, str | None], ...]) +_ROWS: Final = TypeAdapter(tuple[_SubmissionRow, ...]) +_DESCRIPTION: Final = TypeAdapter[str | None](str | None) +_SOURCE: Final = TypeAdapter(dict[str, str]) +_USAGE: Final = TypeAdapter(tuple[_PendingUsage, ...]) +_MAX_PENDING_SUBMISSIONS: Final = 20 +_MAX_PENDING_BYTES: Final = 50 * 1024 * 1024 +_SELECT: Final = """ +SELECT id, plugin_name, operation, status, proposal_json, + archive_content IS NOT NULL AS has_archive, submitted_by, created_at, + base_plugin_id, base_updated_at, reviewed_by, reviewed_at, rejection_reason +FROM "LiteLLM_SkillSubmission" +""" + + +class SubmissionTransaction(PluginPublicationTransaction, Protocol): + @property + def litellm_skillsubmission(self) -> TableActions["prisma_models.LiteLLM_SkillSubmission"]: ... + + +class SubmissionDatabase(SubmissionTransaction, Protocol): + def tx(self) -> AbstractAsyncContextManager[SubmissionTransaction]: ... + + +def _submission(row: "prisma_models.LiteLLM_SkillSubmission") -> SkillSubmission: + return _SubmissionRow.model_validate( + {**row.model_dump(), "has_archive": row.archive_content is not None} + ).submission() + + +class SkillSubmissionRepository: + def __init__(self, prisma_client: "PrismaClient") -> None: + self._database = cast( # cast-ok: Prisma dynamically exposes generated delegates and transactions + SubmissionDatabase, prisma_client.writer_db + ) + + async def get_scan(self, submission_id: str) -> SkillScan | SubmissionMissing | None: + rows: Final = _SCAN_ROWS.validate_python( + await self._database.query_raw( + 'SELECT scan_json FROM "LiteLLM_SkillSubmission" WHERE id = $1', submission_id + ) + ) + if not rows: + return SubmissionMissing() + value: Final = rows[0]["scan_json"] + return None if value is None else SkillScan.model_validate_json(value) + + async def start_scan( + self, submission_id: str, scan: SkillScan + ) -> SkillScanInput | SubmissionMissing | SubmissionConflict: + async with self._database.tx() as transaction: + await transaction.query_raw( + 'SELECT id FROM "LiteLLM_SkillSubmission" WHERE id = $1 FOR UPDATE', submission_id + ) + row: Final = await transaction.litellm_skillsubmission.find_unique(where={"id": submission_id}) + if row is None: + return SubmissionMissing() + if row.status != "pending_review": + return SubmissionConflict("Only pending submissions can be scanned.") + previous: Final = None if row.scan_json is None else SkillScan.model_validate_json(row.scan_json) + if previous is not None and previous.at(scan.started_at).status == "running": + return SubmissionConflict("A scan is already running for this submission.") + stored: Final = _StoredProposal.model_validate_json(row.proposal_json) + source: Final = _SOURCE.validate_python(stored.manifest.get("source", {})) + inherited: Final = ( + await transaction.litellm_claudecodepluginupload.find_unique(where={"plugin_name": row.plugin_name}) + if row.archive_content is None and source.get("source") == "archive" + else None + ) + content: Final = ( + row.archive_content.decode() + if row.archive_content is not None + else verified_archive( + None if inherited is None else inherited.content.decode(), json.dumps(stored.manifest) + ) + ) + managed_archive: Final = urlsplit(source.get("url", "")).path.endswith( + f"/claude-code/plugins/{row.plugin_name}/archive/{source.get('sha256', '')}.zip" + ) + if source.get("source") == "archive" and managed_archive and content is None: + return SubmissionConflict("The proposal's archive is no longer available. Submit a new ZIP to scan it.") + started: Final = scan.model_copy( + update={"archive_sha256": None if content is None else hashlib.sha256(content).hexdigest()} + ) + await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, data={"scan_json": started.model_dump_json()} + ) + return SkillScanInput(started, source, content) + + async def finish_scan(self, submission_id: str, scan: SkillScan) -> None: + async with self._database.tx() as transaction: + rows: Final = _SCAN_ROWS.validate_python( + await transaction.query_raw( + 'SELECT scan_json FROM "LiteLLM_SkillSubmission" WHERE id = $1 FOR UPDATE', submission_id + ) + ) + value: Final = rows[0]["scan_json"] if rows else None + if value is None: + return + current: Final = SkillScan.model_validate_json(value) + if current.id != scan.id or current.status != "running": + return + result: Final = ( + current.at(scan.completed_at) + if scan.completed_at is not None and scan.completed_at >= current.expires_at + else scan + ) + await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, data={"scan_json": result.model_dump_json()} + ) + + async def submit( + self, upload: PluginPublication, submitted_by: str, base: PluginPublicationBase | None + ) -> SkillSubmission | SubmissionLimitExceeded: + from prisma import Base64 + + proposal_json: Final = _StoredProposal(manifest=dict(upload.manifest), version=upload.version).model_dump_json() + incoming_bytes: Final = len(proposal_json.encode("utf-8")) + len(upload.content or b"") + async with self._database.tx() as transaction: + await transaction.query_raw( + "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext('litellm.skill_submissions'), hashtext($1))", + submitted_by, + ) + usage: Final = _USAGE.validate_python( + await transaction.query_raw( + """SELECT COUNT(*) AS count, + COALESCE(SUM(octet_length(proposal_json) + + COALESCE(octet_length(archive_content), 0)), 0) AS storage_bytes + FROM "LiteLLM_SkillSubmission" WHERE submitted_by = $1 AND status = 'pending_review'""", + submitted_by, + ) + )[0] + if usage.count >= _MAX_PENDING_SUBMISSIONS: + return SubmissionLimitExceeded( + "You already have 20 pending skill submissions. " + "Wait for an admin to review one before submitting another." + ) + if usage.storage_bytes + incoming_bytes > _MAX_PENDING_BYTES: + return SubmissionLimitExceeded( + "Pending skill submissions cannot exceed 50 MiB of stored metadata and archives per user. " + "Wait for an admin review or submit less data." + ) + row: Final = await transaction.litellm_skillsubmission.create( + data={ + "plugin_name": upload.name, + "operation": "create" if base is None else "update", + "proposal_json": proposal_json, + "archive_content": None if upload.content is None else Base64.encode(upload.content), + "submitted_by": submitted_by, + "base_plugin_id": None if base is None else base.id, + "base_updated_at": None if base is None else base.updated_at, + } + ) + return _submission(row) + + async def list( + self, submitted_by: str | None, status: SubmissionStatus | None, limit: int, offset: int + ) -> SubmissionPage: + rows: Final = _ROWS.validate_python( + await self._database.query_raw( + _SELECT + + """WHERE ($1::text IS NULL OR submitted_by = $1) + AND ($2::text IS NULL OR status = $2) + ORDER BY created_at DESC, id DESC LIMIT $3 OFFSET $4""", + submitted_by, + status, + limit + 1, + offset, + ) + ) + return SubmissionPage(tuple(row.submission() for row in rows[:limit]), has_more=len(rows) > limit) + + async def get(self, submission_id: str, submitted_by: str | None) -> SkillSubmission | None: + rows: Final = _ROWS.validate_python( + await self._database.query_raw( + _SELECT + "WHERE id = $1 AND ($2::text IS NULL OR submitted_by = $2)", + submission_id, + submitted_by, + ) + ) + return None if not rows else rows[0].submission() + + async def get_archive(self, submission_id: str, submitted_by: str | None) -> bytes | None: + row: Final = await self._database.litellm_skillsubmission.find_first( + where={"id": submission_id, **({"submitted_by": submitted_by} if submitted_by is not None else {})} + ) + return None if row is None or row.archive_content is None else row.archive_content.decode() + + async def review( + self, + submission_id: str, + reviewer: str, + approve: bool, + reason: str | None, + *, + publication_base_url: str | None = None, + ) -> SubmissionReviewResult: + from prisma.errors import UniqueViolationError + + try: + async with self._database.tx() as transaction: + await transaction.query_raw( + 'SELECT id FROM "LiteLLM_SkillSubmission" WHERE id = $1 FOR UPDATE', submission_id + ) + row: Final = await transaction.litellm_skillsubmission.find_unique(where={"id": submission_id}) + if row is None: + return SubmissionMissing() + submission: Final = _submission(row) + if submission.status != "pending_review": + return SubmissionConflict("This submission has already been reviewed.") + content: Final = None if row.archive_content is None else row.archive_content.decode() + if approve and content is not None and not publication_base_url: + return SubmissionConflict("Uploaded skill approval requires a trusted publication URL.") + source: Final = ( + uploaded_plugin_source(publication_base_url, row.plugin_name, hashlib.sha256(content).hexdigest()) + if approve and content is not None and publication_base_url + else _SOURCE.validate_python(submission.proposal.manifest.get("source")) + ) + publication: Final = await _publish_submission(transaction, row, source, content) if approve else None + if isinstance(publication, PluginPublicationError): + return publication + reviewed_at: Final = datetime.now(timezone.utc) + status: Final[SubmissionStatus] = "approved" if approve else "rejected" + await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, + data={ + "status": status, + "reviewed_by": reviewer, + "reviewed_at": reviewed_at, + "rejection_reason": None if approve else reason, + }, + ) + return SubmissionReviewed( + submission=replace( + submission, + status=status, + reviewed_by=reviewer, + reviewed_at=reviewed_at, + rejection_reason=None if approve else reason, + ), + plugin=publication, + published_source=source if approve else None, + ) + except UniqueViolationError: + return SubmissionConflict("A skill with this name is already published. Submit a new update proposal.") + + +async def _publish_submission( + transaction: PluginPublicationTransaction, + row: "prisma_models.LiteLLM_SkillSubmission", + source: Mapping[str, str], + content: bytes | None, +) -> PublishedPlugin | PluginPublicationError: + stored: Final = _StoredProposal.model_validate_json(row.proposal_json) + proposal: Final = stored.proposal(row.plugin_name) + base: Final = ( + None + if row.operation == "create" + else PluginPublicationBase(id=row.base_plugin_id or "", updated_at=row.base_updated_at) + ) + return await publish_plugin( + transaction, + PluginPublication( + name=row.plugin_name, + version=stored.version, + description=proposal.description, + manifest={**stored.manifest, "source": dict(source)}, + content=content, + created_by=row.submitted_by, + ), + base, + ) diff --git a/litellm/repositories/spend_log_insert.py b/litellm/repositories/spend_log_insert.py new file mode 100644 index 00000000000..a958edb9f8a --- /dev/null +++ b/litellm/repositories/spend_log_insert.py @@ -0,0 +1,88 @@ +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError +from typing_extensions import LiteralString + +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + +_COLUMNS: Final[tuple[LiteralString, ...]] = ( + "request_id", + "call_type", + "api_key", + "spend", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "startTime", + "endTime", + "request_duration_ms", + "completionStartTime", + "model", + "model_id", + "model_group", + "custom_llm_provider", + "api_base", + "user", + "metadata", + "cache_hit", + "cache_key", + "request_tags", + "team_id", + "organization_id", + "end_user", + "requester_ip_address", + "messages", + "response", + "session_id", + "status", + "mcp_namespaced_tool_name", + "billing_agent_id", + "agent_id", + "proxy_server_request", + "litellm_call_id", + "created_at", + "updated_at", +) +_JSON_COLUMNS: Final = frozenset({"metadata", "request_tags", "messages", "response", "proxy_server_request"}) + + +class RejectedSpendLogRow(ValueError): + """A row the raw insert cannot encode, so it is isolated like a row Postgres refuses.""" + + +def _json_value(column: str, value: object) -> object: + if isinstance(value, datetime): + return value.isoformat() + if column not in _JSON_COLUMNS or not isinstance(value, str): + return value + try: + return _JSON.validate_json(value) + except ValidationError as error: + raise RejectedSpendLogRow(f"Spend log column {column} is not valid JSON") from error + + +def spend_log_insert_statement(rows: Sequence[Mapping[str, object]]) -> tuple[LiteralString, str]: + if not rows: + raise ValueError("Spend log insert requires at least one row") + fields: Final = frozenset(rows[0]) | {"created_at", "updated_at"} + if not fields.issubset(_COLUMNS) or any(frozenset(row) | {"created_at", "updated_at"} != fields for row in rows): + raise RejectedSpendLogRow("Spend log insert requires matching known columns") + columns: Final = ", ".join('"' + column + '"' for column in _COLUMNS if column in fields) + now: Final = datetime.now(timezone.utc).isoformat() + values: Final = tuple( + {"created_at": now, "updated_at": now, **{column: _json_value(column, value) for column, value in row.items()}} + for row in rows + ) + sql: Final = ( + 'INSERT INTO "LiteLLM_SpendLogs" (' + + columns + + ") SELECT " + + columns + + ' FROM jsonb_populate_recordset(NULL::"LiteLLM_SpendLogs", $1::jsonb) ' + "ON CONFLICT DO NOTHING RETURNING request_id" + ) + return sql, safe_dumps(values, non_finite_as_null=True) diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index 2d3139f7a2c..d010455faa3 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -7,8 +7,15 @@ These are thin wrappers for tables that do not (yet) need domain-specific query methods; richer repositories live in their own modules. """ +from collections.abc import Mapping, Sequence +from datetime import timedelta +from itertools import groupby from typing import TYPE_CHECKING, Any, Final, Generic +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.repositories.accounting_delivery_repository import LedgerDatabase, LedgerTransaction from litellm.repositories.prisma_protocols import RowT_co, TableActions if TYPE_CHECKING: @@ -80,9 +87,117 @@ class OrganizationMembershipRepository(PrismaTableRepository["prisma_models.Lite table_name = "litellm_organizationmembership" +_SPEND_LOG_FIELDS: Final = TypeAdapter(dict[str, object]) + + +def _accounting_payload(value: object) -> Mapping[str, object]: + return ( + _SPEND_LOG_FIELDS.validate_json(value) if isinstance(value, str) else _SPEND_LOG_FIELDS.validate_python(value) + ) + + +_ACCOUNTING_ONLY_FIELDS: Final = frozenset({"accounting_payload", "accounting_log_disabled", "accounting_delivery_id"}) + + +def _settles_once(row: Mapping[str, object]) -> bool: + """A batch or video cost row: update_database gives every other ledger row its own delivery id.""" + return "accounting_delivery_id" not in row + + +def _delivery_id(row: Mapping[str, object]) -> str: + return str(row["accounting_delivery_id"]) if not _settles_once(row) else str(row["request_id"]) + + class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs"]): table_name = "litellm_spendlogs" + async def insert_with_deliveries( + self, rows: Sequence[Mapping[str, object]], *, write_spend_logs: bool = True + ) -> int: + legacy: Final = tuple(row for row in rows if "accounting_payload" not in row) + legacy_count: Final = await self.table.create_many(data=legacy, skip_duplicates=True) if legacy else 0 + ledger: Final = tuple( + sorted( + {_delivery_id(row): row for row in reversed(rows) if "accounting_payload" in row}.values(), + key=_delivery_id, + ) + ) + if not ledger: + return legacy_count + database: Final[LedgerDatabase] = self.prisma_client.writer_db + async with database.tx(timeout=timedelta(seconds=60)) as transaction: + accepted: Final = await self._insert_ledger(transaction, ledger, write_spend_logs=write_spend_logs) + return legacy_count + accepted + + @staticmethod + async def _insert_ledger( + transaction: LedgerTransaction, rows: Sequence[Mapping[str, object]], *, write_spend_logs: bool + ) -> int: + from litellm.proxy.db.db_span import db_span + from litellm.repositories.spend_log_insert import spend_log_insert_statement + + payloads: Final = tuple( + { + "delivery_id": _delivery_id(row), + "request_id": row["request_id"], + "accounting_payload": _accounting_payload(row["accounting_payload"]), + "is_batch": _settles_once(row), + } + for row in rows + ) + inserted_deliveries: Final = await transaction.query_raw( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) ' + "SELECT delivery_id, request_id, accounting_payload, is_batch FROM jsonb_to_recordset($1::jsonb) " + "AS inputs(delivery_id text, request_id text, accounting_payload jsonb, is_batch boolean) " + "ON CONFLICT (delivery_id) DO NOTHING RETURNING delivery_id", + safe_dumps(payloads, non_finite_as_null=True), + ) + accepted_ids: Final = frozenset(str(row["delivery_id"]) for row in inserted_deliveries) + charged_by_legacy: Final = await SpendLogsRepository._jobs_charged_by_legacy( + transaction, + tuple(_delivery_id(row) for row in rows if _settles_once(row) and _delivery_id(row) in accepted_ids), + ) + if charged_by_legacy: + await transaction.execute_raw( + 'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = ANY($1::text[])', list(charged_by_legacy) + ) + accepted: Final = accepted_ids - charged_by_legacy + local_rows: Final = tuple( + {key: value for key, value in row.items() if key not in _ACCOUNTING_ONLY_FIELDS} + for row in rows + if _delivery_id(row) in accepted and row.get("accounting_log_disabled") is not True + ) + if write_spend_logs and local_rows: + groups: Final = tuple( + tuple(group) + for _, group in groupby( + sorted(local_rows, key=lambda row: tuple(sorted(row))), key=lambda row: tuple(sorted(row)) + ) + ) + async with db_span("insert_spend_logs", "LiteLLM_SpendLogs"): + for group in groups: + await transaction.query_raw(*spend_log_insert_statement(group)) + return len(accepted) + + @staticmethod + async def _jobs_charged_by_legacy(transaction: LedgerTransaction, cost_row_ids: Sequence[str]) -> frozenset[str]: + """Batches and videos a pre-ledger poll already charged through their SpendLogs cost row.""" + if not cost_row_ids: + return frozenset() + rows: Final = await transaction.query_raw( + 'SELECT DISTINCT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1::text[]) ' + "AND status = 'success' AND spend > 0", + list(cost_row_ids), + ) + return frozenset(str(row["request_id"]) for row in rows) + + async def has_delivery(self, delivery_id: str) -> bool: + database: Final[LedgerTransaction] = self.prisma_client.writer_db + rows: Final = await database.query_raw( + 'SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = $1', delivery_id + ) + return bool(rows) + class BudgetWindowSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_BudgetWindowSpend"]): table_name = "litellm_budgetwindowspend" @@ -272,3 +387,7 @@ class BackgroundInteractionSettlementRepository( class RetiredAgentRepository(PrismaTableRepository["prisma_models.LiteLLM_RetiredAgent"]): table_name = "litellm_retiredagent" + + +class DailyUserSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyUserSpend"]): + table_name = "litellm_dailyuserspend" diff --git a/litellm/router.py b/litellm/router.py index 4ee138fc9fb..6a60d5d958e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11,7 +11,7 @@ outside code used before the entry points live in ``DeprecatedRouterMembers`` un import inspect import weakref from collections.abc import Mapping -from types import CellType, FunctionType, MappingProxyType +from types import FunctionType, MappingProxyType from typing import TYPE_CHECKING, Final from litellm import router_python_core @@ -35,7 +35,7 @@ from litellm.types.router import ( RetryPolicy, ) -_CORE_SIGNATURE: Final = inspect.signature(PythonCore.__init__) +_CORE_SIGNATURE: Final = inspect.signature(PythonCore.__init__) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # reflection reads only the parameters' names def router_arguments( @@ -65,142 +65,40 @@ else: _replaced_core_members: Final[ # mutable-ok: originals of patched core members, restored on delete dict[tuple[type, str], list[object]] ] = {} - _FACADE_ONLY: Final = frozenset( - { - "__module__", - "__qualname__", - "__doc__", - "__dict__", - "__weakref__", - "__init__", - "__init_subclass__", - "__annotations__", - "__firstlineno__", - "__static_attributes__", - "__classcell__", - "_core_class", - } - ) - - def _with_class_cell(value: object, cell: "CellType") -> object: - """``value`` with its ``__class__`` cell (what zero-argument ``super()`` reads) replaced by ``cell``.""" - if isinstance(value, staticmethod | classmethod): - return type(value)(_function_with_class_cell(value.__func__, cell)) - if isinstance(value, property): - return property( - *( - None if f is None else _function_with_class_cell(f, cell) - for f in (value.fget, value.fset, value.fdel) - ), - value.__doc__, - ) - return _function_with_class_cell(value, cell) - - def _function_with_class_cell(value: object, cell: "CellType") -> object: - """A plain function with its ``__class__`` cell replaced by ``cell``; anything else unchanged.""" - if not isinstance(value, FunctionType) or "__class__" not in value.__code__.co_freevars: - return value - closure: Final = tuple( - cell if name == "__class__" else original - for name, original in zip(value.__code__.co_freevars, value.__closure__ or (), strict=True) - ) - rebound: Final = FunctionType(value.__code__, value.__globals__, value.__name__, value.__defaults__, closure) - rebound.__kwdefaults__ = value.__kwdefaults__ - rebound.__qualname__ = value.__qualname__ - rebound.__dict__.update(value.__dict__) - return rebound - - class _Absent: - """A facade-only member, hidden from the core class: reading it raises the usual AttributeError.""" - - def __set_name__(self, owner: type, name: str) -> None: - self._name = name - - def __get__(self, instance: object, owner: type | None = None) -> object: - if instance is None: - raise AttributeError( - f"type object {getattr(owner, '__name__', owner)!r} has no attribute {self._name!r}" - ) - raise AttributeError(f"{type(instance).__name__!r} object has no attribute {self._name!r}") - - def _core_class_for(facade: type, parent: type) -> type: - """ - A core class carrying a Router subclass's own members, so the core runs its overrides. It - subclasses the facade too, so an override's ``super(Sub, self)`` finds the core it runs on. - """ - cell: Final = CellType() - members: Final = { - name: _with_class_cell(value, cell) for name, value in vars(facade).items() if name not in _FACADE_ONLY - } - namespace: Final = { - # The facade's own plumbing sits in the core class's MRO ahead of the parent core: put - # the parent core's versions back, and hide the facade-only members. - "__init__": parent.__init__, - "__getattribute__": object.__getattribute__, - "__setattr__": parent.__setattr__, - "__delattr__": parent.__delattr__, - "__dir__": parent.__dir__, - **{name: _Absent() for name in ("_attach", "_core", "from_config", "_core_class")}, - **members, - "__module__": facade.__module__, - } - core_class: Final = _CoreType(f"{facade.__name__}Core", (facade, parent), namespace) - cell.cell_contents = core_class - for name in members: - type.__delattr__(facade, name) - return core_class class _RouterType(type): """ - Class-level reads of core members (``Router._helper(...)``) go to the router's core class, and - so do class-level patches, which put the original back when the patch is removed. - ``isinstance`` and ``dir`` see the core class too. + Class-level reads of core members (``Router._helper(...)``) go to ``PythonCore``, and so do + class-level patches, which put the original back when the patch is removed. + ``isinstance`` and ``dir`` see ``PythonCore`` too. """ def __getattr__(cls, name: str) -> object: - if name == "_core_class": - raise AttributeError(name) - return getattr(cls._core_class, name) + return getattr(PythonCore, name) def __setattr__(cls, name: str, value: object) -> None: - core_class: Final = vars(cls).get("_core_class") or getattr(cls, "_core_class", None) - if core_class is None or any(name in vars(base) for base in cls.__mro__) or not hasattr(core_class, name): + if any(name in vars(base) for base in cls.__mro__) or not hasattr(PythonCore, name): super().__setattr__(name, value) return - _replaced_core_members.setdefault((core_class, name), []).append(vars(core_class).get(name, _MISSING)) - setattr(core_class, name, value) + _replaced_core_members.setdefault((PythonCore, name), []).append(vars(PythonCore).get(name, _MISSING)) + setattr(PythonCore, name, value) def __delattr__(cls, name: str) -> None: - core_class: Final = getattr(cls, "_core_class", None) - replaced: Final = _replaced_core_members.get((core_class, name)) if core_class is not None else None + replaced: Final = _replaced_core_members.get((PythonCore, name)) if not replaced or any(name in vars(base) for base in cls.__mro__): super().__delattr__(name) return original: Final = replaced.pop() if original is _MISSING: - delattr(core_class, name) + delattr(PythonCore, name) return - setattr(core_class, name, original) + setattr(PythonCore, name, original) def __instancecheck__(cls, instance: object) -> bool: - return type.__instancecheck__(cls, instance) or isinstance(instance, cls._core_class) + return type.__instancecheck__(cls, instance) or isinstance(instance, PythonCore) def __dir__(cls) -> list[str]: # mutable-ok: dir() returns a list - return sorted({*type.__dir__(cls), *dir(cls._core_class)}) - - class _CoreType(_RouterType): - """The metaclass of a Router subclass's core class: a plain class, none of the facade's redirects.""" - - def __getattr__(cls, name: str) -> object: - raise AttributeError(f"type object {cls.__name__!r} has no attribute {name!r}") - - __setattr__ = type.__setattr__ - __delattr__ = type.__delattr__ - __instancecheck__ = type.__instancecheck__ - __dir__ = type.__dir__ - - def _is_core(instance: object) -> bool: - return isinstance(type(instance), _CoreType) + return sorted({*type.__dir__(cls), *dir(PythonCore)}) def _patchable_method(core: object, name: str) -> bool: """A method of the core: an instance-level replacement of it is also kept on the facade, so @@ -212,27 +110,20 @@ else: return hasattr(type(core), "__getattr__") and inspect.ismethod(getattr(core, name, None)) class Router(DeprecatedRouterMembers, metaclass=_RouterType): - _core_class: type[PythonCore] = PythonCore - - def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super() - if isinstance(cls, _CoreType): - # A core class built below: the parent core's hooks wire it, as for any core class. - super().__init_subclass__(**kwargs) - return - # DeprecatedRouterMembers wires a subclass's overrides of old members; here that runs - # on the core class built from it, which is where the router's own calls land. - super(DeprecatedRouterMembers, cls).__init_subclass__(**kwargs) - type.__setattr__(cls, "_core_class", _core_class_for(cls, cls._core_class)) + def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, refused with the class + raise TypeError( + "litellm.Router cannot be subclassed. Use the router's entry points instead: " + "register_plugin() for custom routing, apply() for settings, observe() for outcomes. " + "See https://docs.litellm.ai/docs/routing_plugins." + ) def __init__(self, *args: object, **kwargs: object) -> None: # kwargs-ok: Router's constructor arguments - self._attach( - make_core(build_router_config(router_arguments(*args, **kwargs)), python_host, type(self)._core_class) - ) + self._attach(make_core(build_router_config(router_arguments(*args, **kwargs)), python_host)) @classmethod def from_config(cls, config: RouterConfig, host: PythonHost = python_host) -> "Router": router: Final = cls.__new__(cls) - router._attach(make_core(config, host, cls._core_class)) + router._attach(make_core(config, host)) return router def _attach(self, core: RouterCore) -> None: @@ -253,8 +144,7 @@ else: try: return object.__getattribute__(self, name) except AttributeError: - if _is_core(self): - raise + pass facade: Final = object.__getattribute__(self, "__dict__") core: Final = facade.get("_core") if core is None: @@ -273,9 +163,6 @@ else: ) from None def __setattr__(self, name: str, value: object) -> None: - if _is_core(self): - super().__setattr__(name, value) - return if _class_descriptor(type(self), name): object.__setattr__(self, name, value) return @@ -289,9 +176,6 @@ else: self.__dict__[name] = value def __delattr__(self, name: str) -> None: - if _is_core(self): - super().__delattr__(name) - return if _class_descriptor(type(self), name): object.__delattr__(self, name) return @@ -299,9 +183,7 @@ else: delattr(self._core, name) def __dir__(self) -> list[str]: # mutable-ok: dir() returns a list - if _is_core(self): - return super().__dir__() - return sorted({*super().__dir__(), *dir(self.__dict__.get("_core", type(self)._core_class))}) + return sorted({*super().__dir__(), *dir(self.__dict__.get("_core", PythonCore))}) Router.__init__.__signature__ = _CORE_SIGNATURE.replace( # pyright: ignore[reportFunctionMemberAccess] # inspect reads it parameters=[parameter for parameter in _CORE_SIGNATURE.parameters.values() if parameter.name != "_host"] diff --git a/litellm/router_api.py b/litellm/router_api.py index f22e9b1f351..82660439ae9 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -32,6 +32,7 @@ as they were. # members are still untyped (bare ``dict`` / ``list``) or mutable (the live objects); the overloads # repeat them unchanged, and typing the members themselves is the core's job. +import copy import hashlib import inspect import json @@ -45,6 +46,7 @@ from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload from litellm.router_config import detached if TYPE_CHECKING: + from functools import _lru_cache_wrapper from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type from opentelemetry.trace import Span @@ -179,9 +181,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id`` set) only by that team, any other deployment by everyone. Needs no router instance. """ - from litellm.router_python_core import PythonCore + from litellm.router_rust.surface.registry import HostRegistry - return PythonCore._core_deployment_usable_by_team(deployment, team_id) + return HostRegistry._core_deployment_usable_by_team(deployment, team_id) def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str: @@ -201,11 +203,77 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str return hashlib.sha256("".join(parts).encode()).hexdigest() +# The arguments ``Router.__init__`` accepts, in its signature's order. Every core reads this list +# (the Python core's ``get_valid_args`` reads its constructor's signature, and the core suite checks +# that the two agree). +ROUTER_INIT_ARGS: Final[tuple[str, ...]] = ( + "model_list", + "assistants_config", + "search_tools", + "guardrail_list", + "redis_url", + "redis_host", + "redis_port", + "redis_password", + "redis_db", + "cache_responses", + "cache_kwargs", + "caching_groups", + "client_ttl", + "polling_interval", + "default_priority", + "num_retries", + "max_fallbacks", + "timeout", + "stream_timeout", + "default_litellm_params", + "default_max_parallel_requests", + "set_verbose", + "debug_level", + "default_fallbacks", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "model_group_alias", + "enable_pre_call_checks", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "plugins", + "retry_after", + "retry_policy", + "model_group_retry_policy", + "allowed_fails", + "allowed_fails_policy", + "cooldown_time", + "disable_cooldowns", + "fallback_on_unsupported_params", + "routing_strategy", + "optional_pre_call_checks", + "routing_strategy_args", + "routing_groups", + "provider_budget_config", + "alerting_config", + "router_general_settings", + "deployment_affinity_ttl_seconds", + "model_group_affinity_config", + "ignore_invalid_deployments", + "enable_health_check_routing", + "health_check_staleness_threshold", + "health_check_ignore_transient_errors", + "background_health_check_model_groups", + "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", + "fallback_access_check", + "fallback_budget_check", + "auto_router_capability_limit", + "session_affinity_repin_on_failover", +) + + def router_init_args() -> tuple[str, ...]: """The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance.""" - from litellm.router_python_core import PythonCore - - return tuple(PythonCore.get_valid_args()) + return ROUTER_INIT_ARGS def mark_advisor_orchestration_failure(exception: BaseException) -> None: @@ -269,9 +337,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob def _json_default_stable_id(obj: object) -> str: - from litellm.router_python_core import PythonCore + from litellm.router_rust.steps.prerouting import _json_default_stable_id as stable_id - return PythonCore._json_default_stable_id(obj) + return stable_id(obj) class RouterRegistry: @@ -316,6 +384,13 @@ class RouterRegistry: return self._router._core_cached_deployment_model_info(id, model_name) return self._router._core_get_deployment_model_info(model_id=id, model_name=model_name) + def deployment_model_info_cache(self) -> "_lru_cache_wrapper[ModelMapInfo | None]": + """ + The lru cache ``deployment_model_info(..., cached=True)`` reads, called as ``(id, model_name)`` + and cleared with ``cache_clear()`` (was the ``cached_deployment_model_info`` attribute). + """ + return self._router._core_cached_deployment_model_info + def router_model_info( self, deployment: "dict[str, object] | Deployment | None", # mutable-ok: the core takes a dict @@ -532,10 +607,8 @@ class RouterRegistry: ``should_include_deployment``), or, without ``model_name``, whether the team may use it at all (was ``deployment_usable_by_team``). """ - from litellm.router_python_core import PythonCore - if model_name is None: - return PythonCore._core_deployment_usable_by_team(deployment, team_id) + return deployment_usable_by_team(deployment, team_id) return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id) @overload @@ -1074,13 +1147,22 @@ PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"}) _MISSING: Final = object() +# The settings fields ``RouterSettings`` copies, so an in-place edit of what a read returns never +# reaches the router (only ``apply()`` changes settings). +_COPIED_SETTING_FIELDS: Final = ( + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "default_litellm_params", +) @dataclass(frozen=True) class RouterSettings: """ The router's configuration as one frozen snapshot (``settings()``). Fields hold the router's - own objects, not copies: mutating a list in here mutates the router's. + own objects, except the fallback lists and ``default_litellm_params``, which are copies: settings + change only through ``apply()``, so editing one of those in place changes nothing on any core. A field the router never set reads as ``None``. Each field has the type of the Router attribute it reads (``_core_``), the type the old @@ -1119,6 +1201,12 @@ class RouterSettings: _router: "Router | None" = field(default=None, repr=False, compare=False) _unset: frozenset[str] = field(default=frozenset(), repr=False, compare=False) + def __post_init__(self) -> None: + for name in _COPIED_SETTING_FIELDS: + value: object = getattr(self, name) # pyright: ignore[reportAny] # a settings field + if isinstance(value, list | dict): + object.__setattr__(self, name, copy.deepcopy(value)) # pyright: ignore[reportUnknownArgumentType] # a settings list or dict + @classmethod def of(cls, router: "Router") -> "RouterSettings": read: Final = {name: getattr(router, "_core_" + name, _MISSING) for name in SETTING_FIELDS} @@ -1188,16 +1276,16 @@ class RouterHealth: @staticmethod def is_blocked(deployment: "Deployment") -> bool: """Whether an admin paused this deployment (was ``is_deployment_blocked``).""" - from litellm.router_python_core import PythonCore + from litellm.router_rust.surface.health_routing import is_blocked as deployment_is_blocked - return PythonCore._core_is_deployment_blocked(deployment) + return deployment_is_blocked(deployment) @staticmethod def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``).""" - from litellm.router_python_core import PythonCore + from litellm.router_rust.surface.health_routing import all_blocked as deployments_all_blocked - return PythonCore._core_are_all_deployments_blocked(deployments) + return deployments_all_blocked(deployments) def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``).""" diff --git a/litellm/router_attempt.py b/litellm/router_attempt.py new file mode 100644 index 00000000000..33e0a19c530 --- /dev/null +++ b/litellm/router_attempt.py @@ -0,0 +1,215 @@ +""" +One router attempt, as the host runs it: the request for the deployment the router core picked. + +The Rust router core picks a deployment and hands the attempt to its host; ``PythonHost`` builds +the keyword arguments for that one deployment here and calls ``litellm.`` with them. Nothing +here selects, retries, falls back or records cooldowns: the core does all of that. Nothing here +reads a Python router either, so this module outlives ``router_python_core.py``. +""" + +import re +import uuid +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter +from litellm.litellm_core_utils.routing_common_utils import without_router_only_kwargs + +# Request keys the router reads to route the call; the deployment call never receives them. +ROUTING_KEYS: Final = frozenset( + { + "model", + "num_retries", + "disable_fallbacks", + "specific_deployment", + "_router_weights", + "_retry_skipped_deployment_ids", + "_target_order", + "model_group_retry_policy", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + } +) + +# The positional parameters of each endpoint, in litellm's order: a call that passes them +# positionally is read back into keywords before it is routed. +POSITIONAL: Final = MappingProxyType( + { + "acompletion": ("model", "messages"), + "completion": ("model", "messages"), + "atext_completion": ("model", "prompt"), + "text_completion": ("model", "prompt"), + "aembedding": ("model", "input"), + "embedding": ("model", "input"), + "aimage_generation": ("prompt", "model"), + "image_generation": ("prompt", "model"), + "atranscription": ("model", "file"), + "aspeech": ("model", "input", "voice"), + "arerank": ("model", "query", "documents"), + "amoderation": ("input", "model"), + "aresponses": ("input", "model"), + "aanthropic_messages": (), + } +) + + +def request_from_call(op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The call's arguments as one keyword request. Raises ``TypeError`` like the endpoint would.""" + names: Final = POSITIONAL.get(op, ()) + if len(args) > len(names): + raise TypeError(f"{op}() takes {len(names)} positional arguments but {len(args)} were given") + request: Final = dict(kwargs) + for name, value in zip(names, args, strict=False): + if name in request: + raise TypeError(f"{op}() got multiple values for argument {name!r}") + request[name] = value + return request + + +# Endpoints the Python router serves through ``_ageneric_api_call_with_fallbacks``, which writes +# the routing metadata under ``litellm_metadata``. +_GENERIC_API_OPS: Final = frozenset({"aanthropic_messages", "aresponses"}) + + +def metadata_field(op: str) -> str: + """``metadata`` or ``litellm_metadata``: where the router writes what it routed the call as.""" + return "litellm_metadata" if op in _GENERIC_API_OPS else get_router_metadata_variable_name(function_name=op) + + +def prepare_call(op: str, request: Mapping[str, object], model_group_alias: str | None) -> Mapping[str, object]: + """What the router adds to a request once, before its first attempt + (``_update_kwargs_before_fallbacks``): a trace id and the model group it was asked for.""" + field: Final = metadata_field(op) + metadata: Final = dict(_as_mapping(request.get(field))) + metadata.update({"model_group": request.get("model"), "model_group_alias": model_group_alias}) + return {**request, "litellm_trace_id": request.get("litellm_trace_id") or str(uuid.uuid4()), field: metadata} + + +def _wildcard_match(pattern: object, requested: str) -> "re.Match[str] | None": + if not isinstance(pattern, str) or "*" not in pattern: + return None + regex: Final = PatternMatchRouter().pattern_to_regex(pattern) + return re.match(regex, requested) or re.match(regex, f"{inferred_provider(requested)}/{requested}") + + +def for_requested_model(deployment: Mapping[str, object], requested: str | None) -> Mapping[str, object]: + """A wildcard deployment as the router serves ``requested`` through it: its ``model`` with + the wildcards filled in from the requested name (``PatternMatchRouter.route``), by the team's + public pattern (``_core_team_pattern_routers``) or its own. Any other deployment comes back + unchanged.""" + params: Final = _as_mapping(deployment.get("litellm_params")) + model: Final = params.get("model") + if requested is None or not isinstance(model, str): + return deployment + public_name: Final = _as_mapping(deployment.get("model_info")).get("team_public_model_name") + match: Final = _wildcard_match(public_name, requested) or _wildcard_match(deployment.get("model_name"), requested) + if match is None: + return deployment + return { + **deployment, + "litellm_params": {**params, "model": PatternMatchRouter.set_deployment_model_name(match, model)}, + } + + +def attempt_kwargs( + op: str, + request: Mapping[str, object], + deployment: Mapping[str, object], + *, + model_group: str, + default_litellm_params: Mapping[str, object], + timeout: float | None, +) -> Mapping[str, object]: + """The keyword arguments for ``litellm.`` against one deployment + (``_update_kwargs_with_deployment`` and the call sites that merge ``litellm_params``).""" + params: Final = _as_mapping(deployment.get("litellm_params")) + model_info: Final = _model_info_for(request, deployment, model_group) + field: Final = metadata_field(op) + metadata: Final = dict(_as_mapping(request.get(field))) + metadata.update( + { + "model_group": model_group, + "deployment": params.get("model"), + "model_info": model_info, + "api_base": params.get("api_base"), + "deployment_model_name": deployment.get("model_name"), + } + ) + tags: Final = _as_list(params.get("tags")) + if tags: + merged: Final = list(_as_list(metadata.get("tags"))) + merged.extend(tag for tag in tags if tag not in merged) + metadata["tags"] = merged + call: Final = {key: value for key, value in request.items() if key not in ROUTING_KEYS} + for key, value in default_litellm_params.items(): + call.setdefault(key, value) + call[field] = metadata + call["model_info"] = model_info + if "timeout" not in call: + deployment_timeout: Final = params.get("timeout") + chosen: Final = deployment_timeout if deployment_timeout is not None else timeout + if chosen is not None: + call["timeout"] = chosen + return without_router_only_kwargs({**params, **call, "model": params.get("model")}) + + +def _model_info_for( + request: Mapping[str, object], deployment: Mapping[str, object], model_group: str +) -> dict[str, object]: # mutable-ok: a fresh dict for the call + """The deployment's ``model_info``, or, for a request carrying its own credentials, the + request-scoped deployment's (``_handle_clientside_credential``): its own stable id, and the + configured deployment's id as ``original_model_id``.""" + from litellm.litellm_core_utils import clientside_credential_handler as clientside + from litellm.router_api import deployment_id + + model_info: Final = dict(_as_mapping(deployment.get("model_info"))) + if not clientside.is_clientside_credential(request_kwargs=dict(request)): # pyright: ignore[reportUnknownMemberType] # an untyped helper + return model_info + dynamic: Final = _as_mapping( + clientside.get_dynamic_litellm_params( # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an untyped helper + litellm_params=dict(_as_mapping(deployment.get("litellm_params"))), request_kwargs=dict(request) + ) + ) + original: Final = model_info.get("id") + model_info["id"] = deployment_id(model_group, dynamic) + model_info["original_model_id"] = original + return model_info + + +def stamp_attempt_error(error: BaseException, deployment: Mapping[str, object]) -> None: + """What the router stamps on a failed attempt's error: the deployment's own ``num_retries`` + (``_set_deployment_num_retries_on_exception``) and its id (``failed_deployment_id``).""" + params: Final = _as_mapping(deployment.get("litellm_params")) + if getattr(error, "num_retries", None) is None and params.get("num_retries") is not None: + try: + error.num_retries = int(str(params["num_retries"])) # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: the router stamps the error in place + except ValueError: + pass + deployment_id: Final = _as_mapping(deployment.get("model_info")).get("id") + if deployment_id and not getattr(error, "failed_deployment_id", None): + try: + error.failed_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: see above + except AttributeError: + pass + + +def _as_mapping(value: object) -> Mapping[str, object]: + if isinstance(value, Mapping): + entries: Final[Mapping[object, object]] = value # pyright: ignore[reportUnknownVariableType] # a request's untyped mapping + return {str(key): item for key, item in entries.items()} + return MappingProxyType({}) + + +def _as_list(value: object) -> list[object]: # mutable-ok: a fresh list + if isinstance(value, list): + items: Final[list[object]] = value # pyright: ignore[reportUnknownVariableType] # a request's untyped list # mutable-ok: read, then copied + return list(items) + return [] diff --git a/litellm/router_config.py b/litellm/router_config.py index 75c79068191..36154af296c 100644 --- a/litellm/router_config.py +++ b/litellm/router_config.py @@ -79,7 +79,12 @@ def detached(value: object) -> object: def build_router_config(arguments: Mapping[str, object]) -> RouterConfig: - """The constructor arguments, detached from the caller's objects: settings are static.""" + """The constructor arguments, detached from the caller's objects: settings are static. + + A bool `cooldown_time` is refused: `True` is not a number of seconds, and + reading it as 1 s or as the default would both be a guess.""" + if isinstance(arguments.get("cooldown_time"), bool): + raise TypeError("Router cooldown_time must be a number of seconds, not a bool") return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()})) diff --git a/litellm/router_core.py b/litellm/router_core.py index acbe82a0643..c077b3f0015 100644 --- a/litellm/router_core.py +++ b/litellm/router_core.py @@ -1,14 +1,16 @@ """ ``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one. -``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core -(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when -the router is built, from its ``RouterConfig``. +``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. ``RustCore`` +(``litellm/router_rust/core.py`` over litellm-rust/crates/router-core) is a separate implementation +that reaches outside itself only through its host; it does not use ``PythonCore``. The core is +picked once, when the router is built, from its ``RouterConfig``. """ from collections.abc import Mapping -from typing import TYPE_CHECKING, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable +from litellm._logging import verbose_router_logger from litellm.router_config import RouterConfig if TYPE_CHECKING: @@ -20,7 +22,6 @@ if TYPE_CHECKING: RouterSettings, RoutingObservers, ) - from litellm.router_python_core import PythonCore from litellm.router_python_host import PythonHost from litellm.types.router import CustomRoutingStrategyBase @@ -50,7 +51,26 @@ class RouterCore(Protocol): def retire(self) -> None: ... -def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore: +def make_core(config: RouterConfig, host: "PythonHost") -> RouterCore: + """ + The Rust core when the route catalog's router rule allows it and the Rust core serves this + config, else the Python core. Decided once, for the router's life, from the config alone: the + Rust core never builds, wraps or falls back to a Python core. + """ from litellm.router_python_core import PythonCore + from litellm.rust_bridge.catalog import Route, RouteContext, decision + from litellm.rust_bridge.configuration import Decision - return (core_class or PythonCore)(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature + chosen: Final = decision(RouteContext(Route.ROUTER)) + if chosen is not Decision.PYTHON: + from litellm.router_rust.core import RustCore + from litellm.rust_bridge._native import RustBridgeDeclined + + try: + return RustCore.build(config, host) # pyright: ignore[reportReturnType] # RustCore serves the RouterCore entry points; other members raise RustCoreUnsupported + except RustBridgeDeclined as declined: + # A config the Rust core cannot take keeps the router on Python while both cores exist. + if chosen is Decision.RUST_REQUIRED: + raise + verbose_router_logger.info("Router served by the Python core: %s", declined) + return PythonCore(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 83a1251d3c0..9d915fe0982 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -5,8 +5,10 @@ DeprecatedRouterMembers: the Router members that outside code used before the en entry points (``registry()``, ``settings()``, ``health()``, ``select()``, ``apply()``, ``observe()``, ``register_plugin()``, ``retire()``; see litellm/router_api.py). Every member here returns what it returned before, with the type it had before, and names its replacement in a -``DeprecationWarning``. Reads go to the same Router implementation (``Router._core_``) the -entry point uses; writes go through ``apply`` / ``aapply`` so the registry version moves. +``DeprecationWarning``. Reads go through that entry point, so they work on either router core +(Python or Rust) and, on the Python core, return the same ``_core_`` object as before. +Settings writes go through ``apply`` / ``aapply`` so the registry version moves; any other old +attribute write sets the core's ``_core_``, which each core keeps where its reads find it. Access from the router itself and from its own modules (``litellm.router*``) is not counted: those are the router's internals and move behind the core interface separately. Every other access @@ -17,21 +19,21 @@ deleted (planned about a month after the core swap). # pyright: reportPrivateUsage=false, reportMissingTypeArgument=false, reportUnknownParameterType=false # pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false # pyright: reportUnknownLambdaType=false -# This shim is part of the router (it reads Router's ``_core_*`` members, as Router's own methods do) +# This shim is part of the router (it writes Router's ``_core_*`` members, as Router's own methods do) # and repeats each old member's signature exactly, untyped parts included (``dict``, ``list``), so # callers keep the types they had. Typing those parts is the core's job, not the shim's. -import contextvars import functools import inspect import sys import warnings from collections import Counter -from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload +from typing import TYPE_CHECKING, Final, Generic, ParamSpec, TypeVar, overload -from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet +from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet, RoutingContext +from litellm.router_strategy.least_busy import picks_reserve if TYPE_CHECKING: from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged @@ -39,6 +41,7 @@ if TYPE_CHECKING: from opentelemetry.trace import Span from litellm.caching.caching import RedisCache + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -58,6 +61,7 @@ if TYPE_CHECKING: from litellm.types.utils import ModelInfo as ModelMapInfo _T: Final = TypeVar("_T") +_P: Final = ParamSpec("_P") DEPRECATED_ROUTER_MEMBER_USES: Final[Counter[str]] = Counter() # mutable-ok: the live tally _note increments @@ -129,6 +133,9 @@ class _DeprecatedAttribute(Generic[_T]): _note(self.name, self.replacement) router: Final = _as_router(obj) try: + if obj.__dict__.get("_core") is None: + # No core yet (a Router made without __init__): the value written through this name. + return getattr(obj, "_core_" + self.name) # pyright: ignore[reportAny] # the pending write, as given return self.read(router) except AttributeError as e: raise _public_attribute_error(router, "_core_" + self.name, self.name, e) from None @@ -164,243 +171,186 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib return _DeprecatedAttribute(replacement, read) -# (id(router), member name) pairs whose subclass override is running in this context. -_OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar( - "router_overrides_running", default=frozenset() -) +def _without_reserving(lookup: Callable[_P, Awaitable[_T]]) -> Callable[_P, Coroutine[object, object, _T]]: + @functools.wraps(lookup) + async def looked_up( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: forwards the wrapped lookup's ParamSpec unchanged + ) -> _T: + with picks_reserve(False): + return await lookup(*args, **kwargs) + + return looked_up -def _override_key(router: object, name: str) -> tuple[int, str]: - """The running-override key for ``router``: a Router and its core share one, so an old member - entered on the Router is seen by the override routing that runs on the core.""" - core: Final = getattr(router, "__dict__", {}).get("_core") - return (id(router if core is None else core), name) - - -def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]: +def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]: """ - A direct call of a subclass override (``router.()``) marks it running, so its - ``super().()`` reaches the parent once instead of re-entering the override through - ``_core_``. + The router's async pick (the old ``async_get_available_deployment`` attribute). A core that + keeps its pick as a member (the Python core's, or one assigned through this attribute) returns + that member, so a patched pick reads back as patched; any other core picks through ``select``. """ - if inspect.iscoroutinefunction(override): - async_override: Final[Callable[..., Awaitable[object]]] = override + member: Final[object] = getattr(router, "_core_async_get_available_deployment", None) + if callable(member): + return member # pyright: ignore[reportReturnType] # the core's own async pick - @functools.wraps(override) - async def entered_async( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged - ) -> object: - token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)}) - try: - return await async_override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) - - return entered_async - - @functools.wraps(override) - def entered( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged + async def async_get_available_deployment( + model: str, + request_kwargs: dict, # mutable-ok: the old parameter type, unchanged + messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged + input: str | list | None = None, # mutable-ok: the old parameter type, unchanged + specific_deployment: bool | None = False, ) -> object: - token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)}) - try: - return override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) + return await router.select( + model, request_kwargs, messages=messages, input=input, specific_deployment=specific_deployment + ) - return entered + return async_get_available_deployment -def _private_alias(name: str, member: Callable[..., object]) -> Callable[..., object]: +def _unreserved_picker(router: "Router") -> Callable[..., Awaitable[object]]: + """``_async_picker``, picking without holding a least-busy reservation (a lookup makes no + call). A pick assigned through the attribute reads back as the object assigned.""" + picker: Final = _async_picker(router) + if inspect.ismethod(picker) or picker.__name__ == "async_get_available_deployment": + return _without_reserving(picker) + return picker + + +def _slack_alerting_logger(router: "Router") -> "SlackAlerting": + """``observe().slack_alerting_logger``; a router without one has no such attribute, as before.""" + logger: Final = router.observe().slack_alerting_logger + if logger is None: + raise AttributeError( + f"'{type(router).__name__}' object has no attribute 'slack_alerting_logger'", + name="slack_alerting_logger", + obj=router, + ) + return logger + + +def _run_ready(step: Coroutine[object, object, _T]) -> _T: """ - A private alias of an old member (``_get_all_deployments``). It was bound to Router's own - method, so it never reached a subclass override of ``name``, and still doesn't. + The result of a coroutine that finishes without suspending: ``select(kind='pass_through')`` + picks synchronously on every core, so the old synchronous member can return its result. """ - return _enter_override(name, member) - - -def _route_override(cls: type, name: str, override: Callable[..., object]) -> Callable[..., object]: - """ - Make the router's own calls to ``_core_`` reach a subclass override of ````. - - When the override calls ``super().()``, the call comes back to ``_core_``; it is - then sent on to the parent's implementation instead of back into the override. - """ - core: Final = "_core_" + name - - def parent(router: object) -> Callable[..., object]: - return getattr(super(cls, router), core) - - if inspect.iscoroutinefunction(override): - async_override: Final[Callable[..., Awaitable[object]]] = override - - @functools.wraps(override) - async def routed_async( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged - ) -> object: - key: Final = _override_key(router, name) - running: Final = _OVERRIDES_RUNNING.get() - if key in running: - parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core) - return await parent_async(*args, **kwargs) - token: Final = _OVERRIDES_RUNNING.set(running | {key}) - try: - return await async_override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) - - return routed_async - - @functools.wraps(override) - def routed( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged - ) -> object: - key: Final = _override_key(router, name) - running: Final = _OVERRIDES_RUNNING.get() - if key in running: - return parent(router)(*args, **kwargs) - token: Final = _OVERRIDES_RUNNING.set(running | {key}) - try: - return override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) - - return routed + try: + step.send(None) + except StopIteration as done: + return done.value # pyright: ignore[reportAny] # the coroutine's own result + step.close() + raise RuntimeError("select(kind='pass_through') suspended; it must pick without awaiting") class DeprecatedRouterMembers: """Old Router members, kept for about a month. Each names its replacement.""" - def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super() - # A Router subclass that overrides an old member (say async_pre_routing_hook) keeps - # having its override called by the router, which now calls _core_ internally. - super().__init_subclass__(**kwargs) - for name, value in list(vars(cls).items()): - if ( - name.startswith("_core_") - or name not in vars(DeprecatedRouterMembers) - or not hasattr(cls, "_core_" + name) - ): - continue - if isinstance(value, staticmethod): - # Its super() call reaches the old member here, which calls Router's own implementation. - setattr(cls, "_core_" + name, value) - elif inspect.isfunction(value): - setattr(cls, "_core_" + name, _route_override(cls, name, value)) - setattr(cls, name, _enter_override(name, value)) - # -- registry: attributes -------------------------------------------------------------------- - model_list = _attr("registry().all()", lambda r: r._core_model_list) - model_names = _attr("registry().names(kind='group_set')", lambda r: r._core_model_names) - deployment_names = _attr("registry().names(kind='deployment_names')", lambda r: r._core_deployment_names) - team_public_model_names = _attr("registry().names(kind='team_public')", lambda r: r._core_team_public_model_names) + model_list = _attr("registry().all()", lambda r: r.registry().all()) + model_names = _attr("registry().names(kind='group_set')", lambda r: r.registry().names(kind="group_set")) + deployment_names = _attr( + "registry().names(kind='deployment_names')", lambda r: r.registry().names(kind="deployment_names") + ) + team_public_model_names = _attr( + "registry().names(kind='team_public')", lambda r: r.registry().names(kind="team_public") + ) model_name_to_deployment_indices = _attr( - "registry().index('model_name')", lambda r: r._core_model_name_to_deployment_indices + "registry().index('model_name')", lambda r: r.registry().index("model_name") ) team_model_to_deployment_indices = _attr( - "registry().index('team_model')", lambda r: r._core_team_model_to_deployment_indices + "registry().index('team_model')", lambda r: r.registry().index("team_model") ) cached_deployment_model_info = _attr( "registry().deployment_model_info(id, model_name, cached=True)", - lambda r: r._core_cached_deployment_model_info, + lambda r: r.registry().deployment_model_info_cache(), ) - model_group_alias = _attr("registry().aliases()", lambda r: r._core_model_group_alias) - pattern_router = _attr("registry().patterns()", lambda r: r._core_pattern_router) - team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r._core_team_pattern_routers) - routing_groups = _attr("registry().routing_groups()", lambda r: r._core_routing_groups) + model_group_alias = _attr("registry().aliases()", lambda r: r.registry().aliases()) + pattern_router = _attr("registry().patterns()", lambda r: r.registry().patterns()) + team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r.registry().patterns(team=True)) + routing_groups = _attr("registry().routing_groups()", lambda r: r.registry().routing_groups()) _routing_groups = routing_groups - auto_routers = _attr("registry().plugins('auto')", lambda r: r._core_auto_routers) - complexity_routers = _attr("registry().plugins('complexity')", lambda r: r._core_complexity_routers) - quality_routers = _attr("registry().plugins('quality')", lambda r: r._core_quality_routers) - adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r._core_adaptive_routers) - routing_plugins = _attr("registry().plugins('routing')", lambda r: r._core_routing_plugins) + auto_routers = _attr("registry().plugins('auto')", lambda r: r.registry().plugins("auto")) + complexity_routers = _attr("registry().plugins('complexity')", lambda r: r.registry().plugins("complexity")) + quality_routers = _attr("registry().plugins('quality')", lambda r: r.registry().plugins("quality")) + adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r.registry().plugins("adaptive")) + routing_plugins = _attr("registry().plugins('routing')", lambda r: r.registry().plugins("routing")) prompt_caching_pin_cache = _attr( - "registry().plugins('prompt_caching_pins')", lambda r: r._core_prompt_caching_pin_cache + "registry().plugins('prompt_caching_pins')", lambda r: r.registry().plugins("prompt_caching_pins") ) # -- health and selection: attributes -------------------------------------------------------- - cache = _attr("health().store", lambda r: r._core_cache) - cooldown_cache = _attr("health().cooldowns", lambda r: r._core_cooldown_cache) - health_state_cache = _attr("health().health_checks", lambda r: r._core_health_state_cache) - deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map) - async_get_available_deployment = _attr( - "select(model, request_kwargs)", - lambda r: r._core_async_get_available_deployment, - ) + cache = _attr("health().store", lambda r: r.health().store) + cooldown_cache = _attr("health().cooldowns", lambda r: r.health().cooldowns) + health_state_cache = _attr("health().health_checks", lambda r: r.health().health_checks) + deployment_latency_map = _attr("health().latency", lambda r: r.health().latency) + async_get_available_deployment = _attr("select(model, request_kwargs)", lambda r: _unreserved_picker(r)) # -- observers ------------------------------------------------------------------------------ - slack_alerting_logger = _attr("observe().slack_alerting_logger", lambda r: r._core_slack_alerting_logger) + slack_alerting_logger = _attr("observe().slack_alerting_logger", _slack_alerting_logger) # -- settings ------------------------------------------------------------------------------- - routing_strategy = _setting("routing_strategy", lambda r: r._core_routing_strategy) - context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r._core_context_window_fallbacks) - content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r._core_content_policy_fallbacks) - default_deployment = _setting("default_deployment", lambda r: r._core_default_deployment) - default_litellm_params = _setting("default_litellm_params", lambda r: r._core_default_litellm_params) + routing_strategy = _setting("routing_strategy", lambda r: r.settings().routing_strategy) + context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r.settings().context_window_fallbacks) + content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r.settings().content_policy_fallbacks) + default_deployment = _setting("default_deployment", lambda r: r.settings().default_deployment) + default_litellm_params = _setting("default_litellm_params", lambda r: r.settings().default_litellm_params) default_max_parallel_requests = _setting( - "default_max_parallel_requests", lambda r: r._core_default_max_parallel_requests + "default_max_parallel_requests", lambda r: r.settings().default_max_parallel_requests + ) + cooldown_time = _setting("cooldown_time", lambda r: r.settings().cooldown_time) + allowed_fails = _setting("allowed_fails", lambda r: r.settings().allowed_fails) + allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r.settings().allowed_fails_policy) + disable_cooldowns = _setting("disable_cooldowns", lambda r: r.settings().disable_cooldowns) + enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r.settings().enable_pre_call_checks) + enable_health_check_routing = _setting( + "enable_health_check_routing", lambda r: r.settings().enable_health_check_routing ) - cooldown_time = _setting("cooldown_time", lambda r: r._core_cooldown_time) - allowed_fails = _setting("allowed_fails", lambda r: r._core_allowed_fails) - allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r._core_allowed_fails_policy) - disable_cooldowns = _setting("disable_cooldowns", lambda r: r._core_disable_cooldowns) - enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r._core_enable_pre_call_checks) - enable_health_check_routing = _setting("enable_health_check_routing", lambda r: r._core_enable_health_check_routing) health_check_ignore_transient_errors = _setting( - "health_check_ignore_transient_errors", lambda r: r._core_health_check_ignore_transient_errors + "health_check_ignore_transient_errors", lambda r: r.settings().health_check_ignore_transient_errors ) background_health_check_model_groups = _setting( - "background_health_check_model_groups", lambda r: r._core_background_health_check_model_groups + "background_health_check_model_groups", lambda r: r.settings().background_health_check_model_groups ) - enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r._core_enable_tag_filtering) - tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r._core_tag_filtering_match_any) - tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r._core_tag_routing_prefix) - router_general_settings = _setting("router_general_settings", lambda r: r._core_router_general_settings) - provider_budget_config = _setting("provider_budget_config", lambda r: r._core_provider_budget_config) + enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r.settings().enable_tag_filtering) + tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r.settings().tag_filtering_match_any) + tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r.settings().tag_routing_prefix) + router_general_settings = _setting("router_general_settings", lambda r: r.settings().router_general_settings) + provider_budget_config = _setting("provider_budget_config", lambda r: r.settings().provider_budget_config) provider_default_deployment_ids = _setting( - "provider_default_deployment_ids", lambda r: r._core_provider_default_deployment_ids + "provider_default_deployment_ids", lambda r: r.settings().provider_default_deployment_ids ) - fallback_access_check = _setting("fallback_access_check", lambda r: r._core_fallback_access_check) - fallback_budget_check = _setting("fallback_budget_check", lambda r: r._core_fallback_budget_check) + fallback_access_check = _setting("fallback_access_check", lambda r: r.settings().fallback_access_check) + fallback_budget_check = _setting("fallback_budget_check", lambda r: r.settings().fallback_budget_check) # These three are declared with the attribute type Router gave them rather than the descriptor's: # protocols in router_utils and the proxy match Router on them, and protocol matching does not # see through a descriptor. Reads and writes still go through _setting at runtime. fallbacks: list | list[dict[str, list[str] | list | None]] | None = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type - "fallbacks", lambda r: r._core_fallbacks + "fallbacks", lambda r: r.settings().fallbacks ) kubernetes_pod_discovery: "KubernetesPodDiscovery" = _setting( # pyright: ignore[reportAssignmentType] # see above - "kubernetes_pod_discovery", lambda r: r._core_kubernetes_pod_discovery + "kubernetes_pod_discovery", lambda r: r.settings().kubernetes_pod_discovery ) search_tools: "list[SearchToolTypedDict]" = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type - "search_tools", lambda r: r._core_search_tools + "search_tools", lambda r: r.settings().search_tools ) - guardrail_list = _setting("guardrail_list", lambda r: r._core_guardrail_list) - cache_responses = _setting("cache_responses", lambda r: r._core_cache_responses) + guardrail_list = _setting("guardrail_list", lambda r: r.settings().guardrail_list) + cache_responses = _setting("cache_responses", lambda r: r.settings().cache_responses) # -- registry: one deployment --------------------------------------------------------------- def get_deployment(self, model_id: str) -> "Deployment | None": _note("get_deployment", "registry().deployment(id)") - return _as_router(self)._core_get_deployment(model_id=model_id) + return _as_router(self).registry().deployment(model_id) def has_model_id(self, candidate_id: str) -> bool: _note("has_model_id", "registry().has_deployment(id)") - return _as_router(self)._core_has_model_id(candidate_id) + return _as_router(self).registry().has_deployment(candidate_id) def get_model_info(self, id: str) -> dict | None: # mutable-ok: the old return type, unchanged _note("get_model_info", "registry().deployment_dict(id)") - return _as_router(self)._core_get_model_info(id=id) + return _as_router(self).registry().deployment_dict(id) def get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None": _note("get_deployment_model_info", "registry().deployment_model_info(id, model_name)") - return _as_router(self)._core_get_deployment_model_info(model_id=model_id, model_name=model_name) + return _as_router(self).registry().deployment_model_info(model_id, model_name) @overload def get_router_model_info( @@ -420,62 +370,56 @@ class DeprecatedRouterMembers: id: str | None = None, ) -> "ModelMapInfo": _note("get_router_model_info", "registry().router_model_info(...)") - router: Final = _as_router(self) - # The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged. - return router._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above - deployment=deployment, # pyright: ignore[reportArgumentType] # see above - received_model_name=received_model_name, - id=id, # pyright: ignore[reportArgumentType] # see above - ) + return _as_router(self).registry().router_model_info(deployment, received_model_name, id) def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: _note("get_discovered_model_info", "registry().discovered_model_info(id)") - return _as_router(self)._core_get_discovered_model_info(model_id) + return _as_router(self).registry().discovered_model_info(model_id) # -- registry: many deployments ------------------------------------------------------------- def get_model_list( self, model_name: str | None = None, team_id: str | None = None ) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged _note("get_model_list", "registry().deployments(model_name, team_id=...)") - return _as_router(self)._core_get_model_list(model_name=model_name, team_id=team_id) + return _as_router(self).registry().deployments(model_name, team_id=team_id) def get_all_deployments( self, model_name: str, model_alias: str | None = None, team_id: str | None = None ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_all_deployments", "registry().deployments(model_name, source='all')") - return _as_router(self)._core_get_all_deployments( - model_name=model_name, model_alias=model_alias, team_id=team_id + return ( + _as_router(self).registry().deployments(model_name, source="all", model_alias=model_alias, team_id=team_id) ) - _get_all_deployments = _private_alias("get_all_deployments", get_all_deployments) + _get_all_deployments = get_all_deployments def config_deployments(self) -> Iterator[Mapping[str, object]]: _note("config_deployments", "registry().deployments(source='config')") - return _as_router(self)._core_config_deployments() + return _as_router(self).registry().deployments(source="config") def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None": _note("deployments_named", "registry().deployments(model_name, source='named')") - return _as_router(self)._core_deployments_named(model_name) + return _as_router(self).registry().deployments(model_name, source="named") def get_model_list_from_model_alias( self, model_name: str | None = None ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_model_list_from_model_alias", "registry().deployments(model_name, source='alias')") - return _as_router(self)._core_get_model_list_from_model_alias(model_name=model_name) + return _as_router(self).registry().deployments(model_name, source="alias") def get_model_list_from_routing_groups(self, model_name: str | None = None) -> "Sequence[DeploymentTypedDict]": _note("get_model_list_from_routing_groups", "registry().deployments(model_name, source='routing_groups')") - return _as_router(self)._core_get_model_list_from_routing_groups(model_name=model_name) + return _as_router(self).registry().deployments(model_name, source="routing_groups") def get_model_list_of_routed_group( self, model_group: str ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')") - return _as_router(self)._core_get_model_list_of_routed_group(model_group) + return _as_router(self).registry().deployments(model_group, source="routed_group") def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: _note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')") - return _as_router(self)._core_wildcard_route_deployments(model, team_id) + return _as_router(self).registry().deployments(model, source="wildcard", team_id=team_id) def get_routable_deployments( self, @@ -486,17 +430,21 @@ class DeprecatedRouterMembers: _note( "get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)" ) - return _as_router(self)._core_get_routable_deployments(model, request_kwargs, call_type) + return ( + _as_router(self) + .registry() + .deployments(model, source="routable", request=request_kwargs, call_type=call_type) + ) def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None": _note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')") - return _as_router(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name) + return _as_router(self).registry().deployments(model_group_name, source="first_in_group") def deployments_for_request( self, model: str, request_kwargs: Mapping[str, object] ) -> "Sequence[DeploymentTypedDict]": _note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)") - return _as_router(self)._core_deployments_for_request(model, request_kwargs) + return _as_router(self).registry().deployments(model, source="for_request", request=request_kwargs) def should_include_deployment( self, @@ -505,135 +453,137 @@ class DeprecatedRouterMembers: team_id: str | None = None, ) -> bool: _note("should_include_deployment", "registry().includes(deployment, model_name=..., team_id=...)") - return _as_router(self)._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id) + return _as_router(self).registry().includes(model, model_name=model_name, team_id=team_id) @staticmethod def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool: _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)") - from litellm.router_python_core import PythonCore + from litellm.router_api import deployment_usable_by_team as usable_by_team - return PythonCore._core_deployment_usable_by_team(model, team_id) + return usable_by_team(model, team_id) _deployment_usable_by_team = deployment_usable_by_team # -- registry: names ------------------------------------------------------------------------- def get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: the old return type, unchanged _note("get_model_names", "registry().names(team_id=...)") - return _as_router(self)._core_get_model_names(team_id=team_id) + return _as_router(self).registry().names(team_id=team_id) def get_model_ids( self, model_name: str | None = None, exclude_team_models: bool = False ) -> list[str]: # mutable-ok: the old return type, unchanged _note("get_model_ids", "registry().names(model_name, kind='ids')") - return _as_router(self)._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models) + return _as_router(self).registry().names(model_name, kind="ids", exclude_team_models=exclude_team_models) def resolved_litellm_models( self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False ) -> tuple[str, ...]: _note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')") - return _as_router(self)._core_resolved_litellm_models( - model_name, team_id=team_id, use_base_model=use_base_model + return ( + _as_router(self) + .registry() + .names(model_name, kind="litellm_base_models" if use_base_model else "litellm_models", team_id=team_id) ) def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: _note("wildcard_routed_models", "registry().names(model, kind='wildcard')") - return _as_router(self)._core_wildcard_routed_models(model, team_id) + return _as_router(self).registry().names(model, kind="wildcard", team_id=team_id) def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: _note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')") - return _as_router(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id) + return _as_router(self).registry().names(model, kind="route_candidates", team_id=team_id) # -- registry: resolving names --------------------------------------------------------------- def routable_model_group(self, model_name: str) -> str: _note("routable_model_group", "registry().resolve(name)") - return _as_router(self)._core_routable_model_group(model_name) + return _as_router(self).registry().resolve(model_name) def get_model_from_alias(self, model: str) -> str | None: _note("get_model_from_alias", "registry().resolve(name, kind='alias')") - return _as_router(self)._core_get_model_from_alias(model=model) + return _as_router(self).registry().resolve(model, kind="alias") - _get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias) + _get_model_from_alias = get_model_from_alias def resolve_served_model_name(self, model_name: str, served_model: str) -> str: _note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)") - return _as_router(self)._core_resolve_served_model_name(model_name, served_model) + return _as_router(self).registry().resolve(model_name, kind="served", served_model=served_model) def resolve_model_name_from_model_id( self, model_id: str | None, custom_llm_provider: str | None = None ) -> str | None: _note("resolve_model_name_from_model_id", "registry().resolve(id, kind='from_id')") - return _as_router(self)._core_resolve_model_name_from_model_id( - model_id, custom_llm_provider=custom_llm_provider - ) + return _as_router(self).registry().resolve(model_id, kind="from_id", custom_llm_provider=custom_llm_provider) def map_team_model( self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None ) -> str | None: _note("map_team_model", "registry().resolve(name, kind='team_model', team_id=...)") - return _as_router(self)._core_map_team_model(team_model_name, team_id, custom_llm_provider=custom_llm_provider) + return ( + _as_router(self) + .registry() + .resolve(team_model_name, kind="team_model", team_id=team_id, custom_llm_provider=custom_llm_provider) + ) def is_recognized_model(self, model: str) -> bool: _note("is_recognized_model", "registry().resolve(name, kind='recognized')") - return _as_router(self)._core_is_recognized_model(model) + return _as_router(self).registry().resolve(model, kind="recognized") def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: _note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')") - return _as_router(self)._core_get_routable_upstream_model(model_name, team_id=team_id) + return _as_router(self).registry().resolve(model_name, kind="upstream", team_id=team_id) def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: _note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')") - return _as_router(self)._core_get_search_tool_support(model_name, team_id=team_id) + return _as_router(self).registry().resolve(model_name, kind="search_tool_support", team_id=team_id) def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged _note("get_model_group", "registry().resolve(id, kind='group_of_id')") - return _as_router(self)._core_get_model_group(id=id) + return _as_router(self).registry().resolve(id, kind="group_of_id") def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: _note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')") - return _as_router(self)._core_is_model_access_group_for_wildcard_route(model_access_group=model_access_group) + return _as_router(self).registry().resolve(model_access_group, kind="wildcard_access_group") - _is_model_access_group_for_wildcard_route = _private_alias( - "is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route - ) + _is_model_access_group_for_wildcard_route = is_model_access_group_for_wildcard_route def team_model_has_alternatives(self, deployment_id: str) -> bool: _note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')") - return _as_router(self)._core_team_model_has_alternatives(deployment_id) + return _as_router(self).registry().resolve(deployment_id, kind="team_model_alternatives") def routing_group_has_alternatives(self, model_group: str | None) -> bool: _note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')") - return _as_router(self)._core_routing_group_has_alternatives(model_group) + return _as_router(self).registry().resolve(model_group, kind="routing_group_alternatives") # -- registry: groups, access groups, credentials, plugins ----------------------------------- def get_model_group_info( self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None ) -> "ModelGroupInfo | None": _note("get_model_group_info", "registry().group(name)") - return _as_router(self)._core_get_model_group_info(model_group, include_hidden=include_hidden, team_id=team_id) + return _as_router(self).registry().group(model_group, include_hidden=include_hidden, team_id=team_id) def cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None": _note("cached_model_group_info", "registry().group(name, field='cached_info')") - return _as_router(self)._core_cached_model_group_info(model_group) + return _as_router(self).registry().group(model_group, field="cached_info") def get_configured_display_name(self, model_name: str) -> str | None: _note("get_configured_display_name", "registry().group(name, field='display_name')") - return _as_router(self)._core_get_configured_display_name(model_name) + return _as_router(self).registry().group(model_name, field="display_name") def get_configured_mode(self, model_name: str) -> str | None: _note("get_configured_mode", "registry().group(name, field='mode')") - return _as_router(self)._core_get_configured_mode(model_name) + return _as_router(self).registry().group(model_name, field="mode") def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: _note("get_configured_service_tiers", "registry().group(name, field='service_tiers')") - return _as_router(self)._core_get_configured_service_tiers(model_name, team_id=team_id) + return _as_router(self).registry().group(model_name, field="service_tiers", team_id=team_id) def get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]: _note("get_configured_token_limits", "registry().group(name, field='token_limits')") - return _as_router(self)._core_get_configured_token_limits(model_name) + return _as_router(self).registry().group(model_name, field="token_limits") def get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None": _note("get_model_listing_info", "registry().group(name, field='listing_info')") - return _as_router(self)._core_get_model_listing_info(model_name) + return _as_router(self).registry().group(model_name, field="listing_info") def get_model_access_groups( self, @@ -642,31 +592,31 @@ class DeprecatedRouterMembers: team_id: str | None = None, ) -> dict[str, list[str]]: # mutable-ok: the old return type, unchanged _note("get_model_access_groups", "registry().access_groups(...)") - return _as_router(self)._core_get_model_access_groups( - model_name=model_name, model_access_group=model_access_group, team_id=team_id + return ( + _as_router(self) + .registry() + .access_groups(model_name=model_name, model_access_group=model_access_group, team_id=team_id) ) def get_deployment_credentials(self, model_id: str) -> dict | None: # mutable-ok: the old return type, unchanged _note("get_deployment_credentials", "registry().credentials(id)") - return _as_router(self)._core_get_deployment_credentials(model_id=model_id) + return _as_router(self).registry().credentials(model_id) def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> "dict[str, Any] | None": # pyright: ignore[reportExplicitAny] # the old return type, unchanged # mutable-ok: same _note("get_deployment_credentials_with_provider", "registry().credentials(id, form='with_provider')") - return _as_router(self)._core_get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id) + return _as_router(self).registry().credentials(model_id, form="with_provider", team_id=team_id) def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None": _note("get_credential_deployment", "registry().credentials(id, form='deployment')") - return _as_router(self)._core_get_credential_deployment(model_id=model_id, team_id=team_id) + return _as_router(self).registry().credentials(model_id, form="deployment", team_id=team_id) def get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None": _note("get_router_deployment_budget_limiter", "registry().plugins('budget_limiter')") - return _as_router(self)._core_get_router_deployment_budget_limiter() + return _as_router(self).registry().plugins("budget_limiter") - _get_router_deployment_budget_limiter = _private_alias( - "get_router_deployment_budget_limiter", get_router_deployment_budget_limiter - ) + _get_router_deployment_budget_limiter = get_router_deployment_budget_limiter @staticmethod def generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: the old parameter type @@ -678,15 +628,15 @@ class DeprecatedRouterMembers: # -- settings: methods ----------------------------------------------------------------------- def get_settings(self) -> dict[str, object]: # mutable-ok: Router._core_get_settings's type, a fresh dict _note("get_settings", "settings().as_dict()") - return _as_router(self)._core_get_settings() + return dict(_as_router(self).settings().as_dict()) def get_allowed_fails_from_policy(self, exception: Exception) -> int | None: _note("get_allowed_fails_from_policy", "settings().allowed_fails_for(exception)") - return _as_router(self)._core_get_allowed_fails_from_policy(exception=exception) + return _as_router(self).settings().allowed_fails_for(exception) def get_available_guardrail(self, guardrail_name: str) -> "GuardrailTypedDict": _note("get_available_guardrail", "settings().guardrail(name)") - return _as_router(self)._core_get_available_guardrail(guardrail_name) + return _as_router(self).settings().guardrail(guardrail_name) # -- health ---------------------------------------------------------------------------------- @staticmethod @@ -709,31 +659,31 @@ class DeprecatedRouterMembers: def get_fully_blocked_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged _note("get_fully_blocked_model_names", "health().fully_blocked_model_names()") - return _as_router(self)._core_get_fully_blocked_model_names() + return _as_router(self).health().fully_blocked_model_names() async def async_get_fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged _note("async_get_fully_unhealthy_model_names", "health().fully_unhealthy_model_names()") - return await _as_router(self)._core_async_get_fully_unhealthy_model_names() + return await _as_router(self).health().fully_unhealthy_model_names() async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: _note("get_model_group_usage", "health().usage(group)") - return await _as_router(self)._core_get_model_group_usage(model_group=model_group) + return await _as_router(self).health().usage(model_group) async def get_remaining_model_group_usage( self, model_group: str ) -> dict[str, int]: # mutable-ok: the old return type, unchanged _note("get_remaining_model_group_usage", "health().usage(group, kind='remaining')") - return await _as_router(self)._core_get_remaining_model_group_usage(model_group=model_group) + return await _as_router(self).health().usage(model_group, kind="remaining") def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": _note("team_ptu_ceiling", "health().team_ptu_ceiling(team_id, group)") - return _as_router(self)._core_team_ptu_ceiling(team_id, model_group) + return _as_router(self).health().team_ptu_ceiling(team_id, model_group) def split_request_priority( self, model: str, kwargs: Mapping[str, object] ) -> tuple[int | None, Mapping[str, object]]: _note("split_request_priority", "health().split_priority(model, kwargs)") - return _as_router(self)._core_split_request_priority(model, kwargs) + return _as_router(self).health().split_priority(model, kwargs) # -- selection ------------------------------------------------------------------------------- async def async_get_healthy_deployments( @@ -747,9 +697,10 @@ class DeprecatedRouterMembers: health_check_probe: bool = False, ) -> list[dict] | dict: # mutable-ok: the old return type, unchanged _note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')") - return await _as_router(self)._core_async_get_healthy_deployments( - model=model, - request_kwargs=request_kwargs, + return await _as_router(self).select( + model, + request_kwargs, + kind="healthy", messages=messages, input=input, specific_deployment=specific_deployment, @@ -757,7 +708,7 @@ class DeprecatedRouterMembers: health_check_probe=health_check_probe, ) - def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from the core + def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from select self, model: str, messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged @@ -766,12 +717,15 @@ class DeprecatedRouterMembers: request_kwargs: dict | None = None, # mutable-ok: the old parameter type, unchanged ): _note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')") - return _as_router(self)._core_get_available_deployment_for_pass_through( - model=model, - messages=messages, - input=input, - specific_deployment=specific_deployment, - request_kwargs=request_kwargs, + return _run_ready( + _as_router(self).select( + model, + request_kwargs, + kind="pass_through", + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) ) # -- writes ---------------------------------------------------------------------------------- @@ -791,7 +745,7 @@ class DeprecatedRouterMembers: _note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))") _as_router(self).apply(ChangeSet(redis_cache=cache)) - _update_redis_cache = _private_alias("update_redis_cache", update_redis_cache) + _update_redis_cache = update_redis_cache async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None: _note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))") @@ -808,8 +762,15 @@ class DeprecatedRouterMembers: def bind_to_request_logging(self, logging_obj: "LiteLLMLogging") -> None: _note("bind_to_request_logging", "observe(logging_obj)") - # Straight to the core, as before: observe() skips binding when logging_obj is None, the old member did not. - _as_router(self)._core_bind_to_request_logging(logging_obj) + router: Final = _as_router(self) + if logging_obj is None: # pyright: ignore[reportUnnecessaryComparison] # callers pass None despite the type + # observe() skips binding when logging_obj is None; the old member bound it, and failed on None. + raise AttributeError( + "'NoneType' object has no attribute 'add_dynamic_function_callbacks'", + name="add_dynamic_function_callbacks", + obj=None, + ) + router.observe(logging_obj) def discard(self) -> None: _note("discard", "retire()") @@ -822,7 +783,7 @@ class DeprecatedRouterMembers: request_kwargs: dict[str, object] | None = None, # mutable-ok: the old parameter type, unchanged ) -> None: _note("arm_routing_read_prefetch", "RoutingContext.arm_read_prefetch") - _as_router(self)._core_arm_routing_read_prefetch(model, request_kwargs) + RoutingContext(_as_router(self)).arm_read_prefetch(model, request_kwargs) async def async_pre_routing_hook( self, @@ -833,7 +794,7 @@ class DeprecatedRouterMembers: specific_deployment: bool | None = False, ) -> "PreRoutingHookResponse | None": _note("async_pre_routing_hook", "RoutingContext.pre_routing_hook") - return await _as_router(self)._core_async_pre_routing_hook( + return await RoutingContext(_as_router(self)).pre_routing_hook( model=model, request_kwargs=request_kwargs, messages=messages, @@ -843,7 +804,7 @@ class DeprecatedRouterMembers: def log_retry(self, kwargs: dict, e: Exception) -> dict: # mutable-ok: the old signature, unchanged _note("log_retry", "RoutingContext.log_retry") - return _as_router(self)._core_log_retry(kwargs=kwargs, e=e) + return RoutingContext(_as_router(self)).log_retry(kwargs=kwargs, e=e) # Router-family functions and classes that took the Router (``litellm_router_instance=``, diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index c0c5d74a88a..8ca2afb9a99 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -127,6 +127,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -282,7 +283,15 @@ from litellm.router_strategy.complexity_router.context_compaction import ( reject_recursive_compactor, surface_for_call, ) -from litellm.router_strategy.least_busy import LeastBusyLoggingHandler +from litellm.router_strategy.least_busy import ( + CALL_IDENTITY, + SELECTION_RESERVATION_MAX_KEYS, + LeastBusyLoggingHandler, + call_id_of, + owning_picks, + pick_reserves, + picks_reserve, +) from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler @@ -314,6 +323,7 @@ from litellm.router_utils.cooldown_handlers import ( mark_deployment_selection_failure, set_cooldown_deployments, ) +from litellm.router_utils.deployment_numbers import refuse_unservable_numbers from litellm.router_utils.embedding_batch_size_check import ( async_log_rejection_before_provider_call, clear_rejected_before_provider_call, @@ -358,6 +368,11 @@ from litellm.router_utils.fallback_event_handlers import ( routed_deployment_id, run_async_fallback, ) +from litellm.router_utils.global_callback_holders import ( + live_check_owners, + routers_holding_global_callbacks, + routers_holding_global_callbacks_lock, +) from litellm.router_utils.handle_error import ( async_raise_no_deployment_exception, send_llm_exception_alert, @@ -1225,8 +1240,9 @@ class RoutingArgs(enum.Enum): # as the per-request one built from a caller-supplied user_config, drops out on its # own rather than leaving entries behind that nothing can withdraw. _live_routers: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet() -_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet() -_routers_holding_global_callbacks_lock: Final = threading.Lock() +# Shared with the other router cores of the process (litellm/router_utils/global_callback_holders.py). +_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = routers_holding_global_callbacks # pyright: ignore[reportAssignmentType] # every member serves the holder members +_routers_holding_global_callbacks_lock: Final = routers_holding_global_callbacks_lock def _replay_live_router_model_cost() -> None: @@ -1462,13 +1478,14 @@ class PythonCore(DeprecatedRouterMembers): (``pass_through``). """ if kind == "pick": - return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts - model=model, - request_kwargs=request_kwargs if request_kwargs is not None else {}, - messages=messages, - input=input, - specific_deployment=specific_deployment, - ) + with picks_reserve(False): + return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + request_kwargs=request_kwargs if request_kwargs is not None else {}, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) if kind == "healthy": return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts model=model, @@ -1480,13 +1497,14 @@ class PythonCore(DeprecatedRouterMembers): health_check_probe=health_check_probe, ) if kind == "pass_through": - return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts - model=model, - messages=messages, - input=input, - specific_deployment=specific_deployment, - request_kwargs=request_kwargs, - ) + with picks_reserve(False): + return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + messages=messages, + input=input, + specific_deployment=specific_deployment, + request_kwargs=request_kwargs, + ) raise ValueError(f"unknown select kind {kind!r}") def apply(self, changes: ChangeSet) -> ApplyResult: @@ -1768,6 +1786,9 @@ class PythonCore(DeprecatedRouterMembers): self._core_cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. + self._core_least_busy_reservations: InMemoryCache = InMemoryCache( + max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS + ) self._claude_code_session_router_cache: DualCache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache(), @@ -1865,6 +1886,7 @@ class PythonCore(DeprecatedRouterMembers): # cost-map entries from the list it is serving at that moment, whether # those deployments came from model_list or were added later from the DB. _live_routers.add(self) + live_check_owners.add(self) if allowed_fails is not None: self._core_allowed_fails = allowed_fails @@ -2090,6 +2112,7 @@ class PythonCore(DeprecatedRouterMembers): # Stop contributing to cost-map rebuilds straight away rather than waiting # for this router to be collected. _live_routers.discard(self) + live_check_owners.discard(self) self.retire_routing_state() self._release_global_callbacks() @@ -2271,7 +2294,9 @@ class PythonCore(DeprecatedRouterMembers): selector: RouterStrategySelector | None = None match self._normalize_strategy(strategy): case RoutingStrategy.LEAST_BUSY.value: - selector = LeastBusyLoggingHandler(router_cache=self._core_cache) + selector = LeastBusyLoggingHandler( + router_cache=self._core_cache, reservation_store=self._core_least_busy_reservations + ) case RoutingStrategy.USAGE_BASED_ROUTING.value: selector = LowestTPMLoggingHandler( router_cache=self._core_cache, @@ -2736,6 +2761,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, request_kwargs: dict | None, + reserve: bool, ) -> Any | None: """ Asks the strategy selector for a deployment. Caller handles @@ -2749,6 +2775,8 @@ class PythonCore(DeprecatedRouterMembers): return await selector.async_get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), ) case "usage-based-routing": # `LowestTPMLoggingHandler` (v1) only exposes the sync @@ -2789,6 +2817,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, request_kwargs: dict | None, + reserve: bool, ) -> Any | None: """ Sync sibling of `_select_deployment_async`. Caller handles @@ -2805,6 +2834,8 @@ class PythonCore(DeprecatedRouterMembers): return selector.get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), ) case "usage-based-routing" | "usage-based-routing-v2": return selector.get_available_deployments( @@ -3393,7 +3424,7 @@ class PythonCore(DeprecatedRouterMembers): self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] if any( router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or [])) - for router in tuple(_live_routers) + for router in (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore))) ): return for cb in tuple(self._host.callback_list("callbacks")): @@ -3524,12 +3555,13 @@ class PythonCore(DeprecatedRouterMembers): ) # pick the one that is available (lowest TPM/RPM) - deployment = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(True): + deployment = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._drop_unsupported_classifier_reasoning_effort( deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment model=model, @@ -6104,12 +6136,13 @@ class PythonCore(DeprecatedRouterMembers): deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _image_generation()- model: %s", model) - deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init - model=model, - messages=[{"role": "user", "content": "prompt"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(True): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + messages=[{"role": "user", "content": "prompt"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() @@ -6496,37 +6529,38 @@ class PythonCore(DeprecatedRouterMembers): **kwargs, ): messages: Final = [{"role": "user", "content": prompt}] - try: - kwargs["model"] = model - kwargs["prompt"] = prompt - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + kwargs["model"] = model + kwargs["prompt"] = prompt + kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) + kwargs.setdefault("metadata", {}).update({"model_group": model}) - # pick the one that is available (lowest TPM/RPM) - with endpoint_selection("text_completion", model=model): - deployment: Final = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, + # pick the one that is available (lowest TPM/RPM) + with endpoint_selection("text_completion", model=model), picks_reserve(True): + deployment: Final = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + + data: Final = deployment["litellm_params"].copy() + for k, v in self._core_default_litellm_params.items(): + if k not in kwargs: # prioritize model-specific params > default router params + kwargs[k] = v + elif k == "metadata": + kwargs[k].update(v) + pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults + + # call via litellm.completion() + return self._host.invoke("text_completion")( + **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} ) - - data: Final = deployment["litellm_params"].copy() - for k, v in self._core_default_litellm_params.items(): - if k not in kwargs: # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) - pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) - if pinned_model_id is not None: - kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults - - # call via litellm.completion() - return self._host.invoke("text_completion")( - **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} - ) - except Exception as e: - raise e + except Exception as e: + raise e async def atext_completion( self, @@ -7126,6 +7160,8 @@ class PythonCore(DeprecatedRouterMembers): or self._stream_can_fall_back(model, initial_kwargs) ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline + # An error this loop raises off a frame, not the provider's. + raised_here: BaseException | None = None # rebind-ok: set when the loop raises off a frame try: async for chunk in source_iterator: if _anthropic_stream_forwards_ping_live(chunk, has_generated_content): @@ -7156,25 +7192,27 @@ class PythonCore(DeprecatedRouterMembers): and self._refusal_fallback_available(model, initial_kwargs) ): refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) - raise MidStreamFallbackError( + raised_here = MidStreamFallbackError( message=refusal_error.message, model=model, llm_provider="anthropic", original_exception=refusal_error, is_pre_first_chunk=True, ) + raise raised_here if not has_generated_content and error_event is None: buffered_lifecycle_chunks = (*buffered_lifecycle_chunks, chunk) continue if recoverable_frame_error is not None: assert error_event is not None - raise MidStreamFallbackError( + raised_here = MidStreamFallbackError( message=error_event[1], model=model, llm_provider="anthropic", original_exception=recoverable_frame_error, is_pre_first_chunk=True, ) + raise raised_here for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk buffered_lifecycle_chunks = () @@ -7182,6 +7220,8 @@ class PythonCore(DeprecatedRouterMembers): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk except Exception as stream_error: # noqa: BLE001 # any raised provider error must reach the fallback gate + if stream_error is not raised_here: + await self._areport_raised_messages_stream_failure(stream_error, initial_kwargs, source_iterator) async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, @@ -7201,6 +7241,52 @@ class PythonCore(DeprecatedRouterMembers): wrapper: Final = FallbackAwareAnthropicMessagesStream(stream_with_fallbacks(), source_iterator) return wrapper + async def _areport_raised_messages_stream_failure( + self, stream_error: Exception, attempt_kwargs: Mapping[str, object], source: object + ) -> None: + """A /v1/messages stream whose provider iterator raised, counted against the deployment + that served it and cooled down by the rules, the same as a non-stream call's failure, so a + retry picks another deployment. A stream that logs itself (litellm's own stream logging + reaches the router's failure callback), or a failure the request's logging object already + logged, is not counted twice.""" + from litellm.exceptions import MidStreamFallbackError + from litellm.litellm_core_utils.litellm_logging import Logging + + if isinstance(getattr(source, "logging_obj", None), Logging): + return + logging_obj: Final = attempt_kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, Logging) and logging_obj.model_call_details.get("has_logged_async_failure") is True: + return + bucket: Final = as_str_mapping(attempt_kwargs.get(get_metadata_variable_name_from_kwargs(attempt_kwargs))) or {} + model_info: Final = as_str_mapping(bucket.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + if deployment_id is None: + return + deployment: Final = self.get_deployment(model_id=str(deployment_id)) + deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} + provider_error: Final = ( + stream_error.original_exception + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None + else stream_error + ) + now: Final = datetime.now() # noqa: DTZ005 # naive, as the logging pipeline's times + logged: Final = { + "call_type": "anthropic_messages", + "exception": provider_error, + "litellm_params": { + **{ + name: deployment_params[name] for name in ("api_base", "cooldown_time") if name in deployment_params + }, + "metadata": dict(bucket), + "model_info": dict(model_info), + }, + } + try: + _ = self.deployment_callback_on_failure(logged, None, now, now) # pyright: ignore[reportUnknownMemberType] # an untyped callback + await self.async_deployment_callback_on_failure(logged, None, now, now) # pyright: ignore[reportUnknownMemberType] # see above + except Exception as error: # noqa: BLE001 # a recording never fails the stream + verbose_router_logger.warning("a stream's failure was not recorded: %s", error) + async def _aanthropic_messages_recover_stream_error( self, stream_error: Exception, @@ -7671,64 +7757,69 @@ class PythonCore(DeprecatedRouterMembers): handler_name: Final = original_function.__name__ metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked - try: - verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) - self._update_kwargs_before_fallbacks( - model=model, - kwargs=kwargs, - metadata_variable_name=metadata_variable_name, - ) - with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): - deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) + self._update_kwargs_before_fallbacks( model=model, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, + kwargs=kwargs, + metadata_variable_name=metadata_variable_name, + ) + with ( + self._video_call_pin(handler_name, kwargs), + self._route_capability_pin(handler_name), + picks_reserve(True), + ): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + self._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="generic_api_call", + splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) - self._update_kwargs_with_deployment( - deployment=deployment, - kwargs=kwargs, - function_name="generic_api_call", - splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, - ) - data: Final = deployment["litellm_params"].copy() - model_name: Final = data["model"] + data: Final = deployment["litellm_params"].copy() + model_name: Final = data["model"] - self.total_calls[model_name] += 1 + self.total_calls[model_name] += 1 - # For passthrough routes, use the actual model from deployment - # and swap model name in endpoint if present - if "endpoint" in kwargs and kwargs["endpoint"]: - kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) - kwargs["model"] = model_name + # For passthrough routes, use the actual model from deployment + # and swap model name in endpoint if present + if "endpoint" in kwargs and kwargs["endpoint"]: + kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + kwargs["model"] = model_name - # Perform pre-call checks for routing strategy - self.routing_strategy_pre_call_checks(deployment=deployment) + # Perform pre-call checks for routing strategy + self.routing_strategy_pre_call_checks(deployment=deployment) - custom_llm_provider: Final = provider_for_generic_call(data) + custom_llm_provider: Final = provider_for_generic_call(data) - response: Final = original_function( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self._core_cache_responses, - **kwargs, - } - ) + response: Final = original_function( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self._core_cache_responses, + **kwargs, + } + ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) - return response - except Exception as e: - verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) - if model is not None: - self.fail_calls[model] += 1 - self._stamp_failed_attempt( - e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None - ) - raise e + self.success_calls[model_name] += 1 + verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) + return response + except Exception as e: + verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) + if model is not None: + self.fail_calls[model] += 1 + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) + raise e def embedding( self, @@ -7752,7 +7843,7 @@ class PythonCore(DeprecatedRouterMembers): deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside embedding()- model: %s", model) - with embedding_deployment_selection(): + with embedding_deployment_selection(), picks_reserve(True): deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init model=model, input=input, @@ -8061,68 +8152,71 @@ class PythonCore(DeprecatedRouterMembers): Returns: VectorStoreCreateResponse """ - try: - # If model is None, use the factory function approach (direct SDK call) - if model is None: - from litellm.vector_stores.main import acreate + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + # If model is None, use the factory function approach (direct SDK call) + if model is None: + from litellm.vector_stores.main import acreate - # Use the factory function to handle the call - factory_fn: Final = self.factory_function(acreate, call_type="avector_store_create") - return await factory_fn(**kwargs) + # Use the factory function to handle the call + factory_fn: Final = self.factory_function(acreate, call_type="avector_store_create") + return await factory_fn(**kwargs) - from litellm.vector_stores import acreate as avector_store_create_sdk + from litellm.vector_stores import acreate as avector_store_create_sdk - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "vector-store-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) - data: Final = deployment["litellm_params"].copy() - model_name: Final = data["model"] - self._update_kwargs_with_deployment( - deployment=deployment, - kwargs=kwargs, - function_name="avector_store_create", - ) - - model_client: Final = self._get_async_openai_model_client( - deployment=deployment, - kwargs=kwargs, - ) - self.total_calls[model_name] += 1 - - # Get custom provider from deployment params - custom_llm_provider = data.get("custom_llm_provider") - _, inferred_custom_llm_provider, _, _ = get_llm_provider( - model=data["model"], - custom_llm_provider=custom_llm_provider, - ) - custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - - async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await avector_store_create_sdk( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self._core_cache_responses, - "client": model_client, - **kwargs, - } + parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) + deployment: Final = await self._core_async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "vector-store-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + data: Final = deployment["litellm_params"].copy() + model_name: Final = data["model"] + self._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="avector_store_create", ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + model_client: Final = self._get_async_openai_model_client( + deployment=deployment, + kwargs=kwargs, + ) + self.total_calls[model_name] += 1 - return response - except Exception as e: - verbose_router_logger.exception( - "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e - ) - if model is not None: - self.fail_calls[model] += 1 - raise e + # Get custom provider from deployment params + custom_llm_provider = data.get("custom_llm_provider") + _, inferred_custom_llm_provider, _, _ = get_llm_provider( + model=data["model"], + custom_llm_provider=custom_llm_provider, + ) + custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider + + async with self._deployment_slot( + deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span + ): + response = await avector_store_create_sdk( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self._core_cache_responses, + "client": model_client, + **kwargs, + } + ) + + self.success_calls[model_name] += 1 + verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + + return response + except Exception as e: + verbose_router_logger.exception( + "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) + if model is not None: + self.fail_calls[model] += 1 + raise e def _override_vector_store_methods_for_router(self): """ @@ -8370,12 +8464,13 @@ class PythonCore(DeprecatedRouterMembers): try: verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "batch-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(False): + deployment: Final = await self._core_async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "batch-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id") data: Final = deployment["litellm_params"].copy() @@ -9520,38 +9615,39 @@ class PythonCore(DeprecatedRouterMembers): mock_timeout: Final = kwargs.pop("mock_timeout", None) - try: - self._handle_mock_testing_fallbacks( - kwargs=kwargs, - model_group=model_group, - fallbacks=fallbacks, - context_window_fallbacks=context_window_fallbacks, - content_policy_fallbacks=content_policy_fallbacks, - ) + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + self._handle_mock_testing_fallbacks( + kwargs=kwargs, + model_group=model_group, + fallbacks=fallbacks, + context_window_fallbacks=context_window_fallbacks, + content_policy_fallbacks=content_policy_fallbacks, + ) - if mock_timeout is not None: - response = await self.async_function_with_retries(*args, **kwargs, mock_timeout=mock_timeout) - else: - response = await self.async_function_with_retries(*args, **kwargs) - if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug("Async Response: %s", response) - response = add_fallback_headers_to_response( - response=response, - attempted_fallbacks=0, - ) - return response - except Exception as e: - return await self.async_function_with_fallbacks_common_utils( - e, - disable_fallbacks, - fallbacks, - context_window_fallbacks, - content_policy_fallbacks, - model_group, - args, - kwargs, - include_fallback_errors=include_fallback_errors, - ) + if mock_timeout is not None: + response = await self.async_function_with_retries(*args, **kwargs, mock_timeout=mock_timeout) + else: + response = await self.async_function_with_retries(*args, **kwargs) + if verbose_router_logger.isEnabledFor(logging.DEBUG): + verbose_router_logger.debug("Async Response: %s", response) + response = add_fallback_headers_to_response( + response=response, + attempted_fallbacks=0, + ) + return response + except Exception as e: + return await self.async_function_with_fallbacks_common_utils( + e, + disable_fallbacks, + fallbacks, + context_window_fallbacks, + content_policy_fallbacks, + model_group, + args, + kwargs, + include_fallback_errors=include_fallback_errors, + ) def _handle_mock_testing_fallbacks( self, @@ -10856,6 +10952,7 @@ class PythonCore(DeprecatedRouterMembers): messages=None, input=None, request_kwargs=routing_kwargs, + reserve=False, ) if selected is None: no_deployment: Final = await async_raise_no_deployment_exception( @@ -11200,6 +11297,7 @@ class PythonCore(DeprecatedRouterMembers): model_info=_model_info, ) ) + refuse_unservable_numbers(deployment.to_json(exclude_none=True, warnings=False)) # pyright: ignore[reportUnknownMemberType] # pydantic's to_json PythonCore._register_deployment_pricing(deployment) ## Check if LLM Deployment is allowed for this deployment @@ -11703,6 +11801,16 @@ class PythonCore(DeprecatedRouterMembers): return model_info_is_active_for_environment(model_info=deployment.model_info) def set_model_list(self, model_list: list): + """Replace every deployment. All or nothing: when an entry raises, the deployments served + before the call are put back and the error is raised.""" + previous: Final = copy.deepcopy(list(self.__dict__.get("_core_model_list") or ())) + try: + self._replace_model_list(model_list) # pyright: ignore[reportUnknownMemberType] # the legacy untyped model list + except Exception: + self._replace_model_list(previous) # pyright: ignore[reportUnknownMemberType] # see above + raise + + def _replace_model_list(self, model_list: list) -> None: # pyright: ignore[reportUnknownParameterType, reportMissingTypeArgument] # the legacy untyped model list original_model_list: Final = copy.deepcopy(model_list) self._discovered_model_info_cache.flush_cache() with self._deployment_index_lock: @@ -11996,6 +12104,7 @@ class PythonCore(DeprecatedRouterMembers): # add to model list _deployment: Final = served.to_json(exclude_none=True) + refuse_unservable_numbers(_deployment) # initialize client self._add_deployment(deployment=served) @@ -12139,6 +12248,9 @@ class PythonCore(DeprecatedRouterMembers): _deployment_model_id: Final = deployment.model_info.id or "" _deployment_on_router: Final[Deployment | None] = self._core_get_deployment(model_id=_deployment_model_id) served: Final = without_model_capability_flags(deployment) + # `_create_deployment` lists the model name before it can still fail; a dropped deployment + # takes its name back off the list. + names_before: Final = len(self._core_deployment_names) try: if _deployment_on_router is not None: # deployment with this model_id exists on the router @@ -12188,6 +12300,7 @@ class PythonCore(DeprecatedRouterMembers): deployment.model_info.id, e, ) + del self._core_deployment_names[names_before:] self._restore_deployment_after_failed_upsert( previous_deployment=_deployment_on_router, model_id=_deployment_model_id ) @@ -14678,9 +14791,7 @@ class PythonCore(DeprecatedRouterMembers): """ if team_id is not None and self._team_deployments_named(team_id, model_group): return self._core_get_all_deployments(model_name=model_group, team_id=team_id) - own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns - list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group) - ) + own: Final = self._get_all_deployments(model_name=model_group) named: Final = [*own, *self._core_get_model_list_from_routing_groups(model_name=model_group)] return named or self._get_wildcard_deployments(model_name=model_group) @@ -15888,10 +15999,15 @@ class PythonCore(DeprecatedRouterMembers): # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN THE TEAM ## THIS PREVENTS WRITING FILES OF OTHER TEAMS TO MODELS THAT ARE TEAM-ONLY MODELS + by_id: Final = healthy_deployments if isinstance(healthy_deployments, dict) else None # pyright: ignore[reportUnknownVariableType] # the legacy deployment dict healthy_deployments = filter_team_based_models( healthy_deployments=healthy_deployments, request_kwargs=request_kwargs, ) + if by_id is not None and not healthy_deployments: + # Another team's deployment asked for by id: refused under its group's name. + group_name: Final = str(by_id.get("model_name")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # see above + model = group_name # rebind-ok: the error names the group if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug("healthy_deployments after team filter: %s", healthy_deployments) @@ -16214,6 +16330,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, specific_deployment=specific_deployment, + reserve=pick_reserves(default=True), ) async def _async_get_available_deployment( @@ -16223,6 +16340,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, specific_deployment: bool | None, + reserve: bool, ): if ( self._core_routing_strategy != "usage-based-routing-v2" @@ -16319,6 +16437,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=reserve and request_kwargs not in (None, {}), ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( @@ -16488,6 +16607,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: @@ -17197,6 +17317,16 @@ class PythonCore(DeprecatedRouterMembers): model, ) ) + # As the async pick: a team-owned deployment serves only its own team's callers, also when + # asked for by id (then refused under its group's name). + by_id: Final = healthy_deployments if isinstance(healthy_deployments, dict) else None # pyright: ignore[reportUnknownVariableType] # the legacy deployment dict + healthy_deployments = filter_team_based_models( # rebind-ok: the team filter narrows # pyright: ignore[reportUnknownVariableType] # legacy selector type + healthy_deployments=healthy_deployments, # pyright: ignore[reportArgumentType] # legacy selector type + request_kwargs=request_kwargs, + ) + if by_id is not None and not healthy_deployments: + group_name: Final = str(by_id.get("model_name")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # see above + model = group_name # rebind-ok: the error names the group strategy, strategy_selector = self._get_routing_context(model, request_kwargs) routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs) @@ -17365,6 +17495,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: @@ -17545,6 +17676,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: diff --git a/litellm/router_python_host.py b/litellm/router_python_host.py index 8156740144e..8ab2c06b82f 100644 --- a/litellm/router_python_host.py +++ b/litellm/router_python_host.py @@ -7,7 +7,10 @@ come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the sam """ import asyncio -from collections.abc import Awaitable, Callable, Sequence +import importlib +import time +import types +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import ( TYPE_CHECKING, Final, @@ -19,12 +22,13 @@ from typing import ( from typing_extensions import overload import litellm -from litellm.router_config import RouterDynamicConfig, live_dynamic_config +from litellm.router_config import RouterConfig, RouterDynamicConfig, live_dynamic_config if TYPE_CHECKING: from pydantic import BaseModel - from litellm.caching.caching import Cache + from litellm.caching.caching import Cache, RedisCache + from litellm.caching.dual_cache import DualCache from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper @@ -97,7 +101,7 @@ class PythonHost: def invoke(self, op: str) -> Endpoint: ... def invoke(self, op: str) -> Endpoint: """The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time.""" - endpoint: Final[object] = getattr(litellm, op, None) + endpoint: Final[object] = getattr(litellm, op, None) if "." not in op else _dotted(op) if endpoint is None: raise AttributeError(f"litellm has no endpoint {op!r}") if not callable(endpoint): @@ -107,6 +111,10 @@ class PythonHost: def sleep(self, seconds: float) -> Awaitable[None]: return asyncio.sleep(seconds) + def sleep_sync(self, seconds: float) -> None: + """The backoff of a sync endpoint's retry.""" + time.sleep(seconds) + def dynamic(self) -> RouterDynamicConfig: """The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot).""" return cast(RouterDynamicConfig, live_dynamic_config) # cast-ok: the live view reads the same fields @@ -116,7 +124,7 @@ class PythonHost: return litellm.logging_callback_manager def callback_list(self, name: CallbackList) -> list[object]: # mutable-ok: the process callback list itself - value: Final = getattr(litellm, name) + value: Final[object] = getattr(litellm, name) # pyright: ignore[reportAny] # an untyped litellm global return cast(list[object], value) # cast-ok: the process callback list is an untyped litellm global def set_callback_list(self, name: CallbackList, value: list[object]) -> None: # mutable-ok: assigned as given @@ -131,5 +139,78 @@ class PythonHost: def suppress_debug_info(self) -> None: litellm.suppress_debug_info = True + def routing_cache(self, config: RouterConfig) -> "DualCache": + """The store a router keeps its routing state in (cooldowns, fail counts, usage): in memory, + plus Redis when the router's arguments name one. The host builds and owns it; a Rust core + reads and writes it through cache steps.""" + from litellm.caching.caching import DualCache, InMemoryCache + + return DualCache(redis_cache=redis_cache_for(config), in_memory_cache=InMemoryCache()) + + +def redis_cache_for(config: RouterConfig) -> "RedisCache | None": + """The Redis cache the router's ``redis_*`` and ``cache_kwargs`` arguments describe, or None.""" + import json + + from litellm.caching.caching import RedisCache, RedisClusterCache + from litellm.utils import get_secret + + url: Final = config.get("redis_url") + host: Final = config.get("redis_host") + port: Final = config.get("redis_port") + if url is None and (host is None or port is None): + return None + settings: Final[dict[str, object]] = {} # mutable-ok: built here, then passed as keywords + if url is not None: + settings["url"] = url + if host is not None: + settings["host"] = host + if port is not None: + settings["port"] = str(port) + if config.get("redis_password") is not None: + settings["password"] = config.get("redis_password") + if config.get("redis_db") is not None: + settings["db"] = str(config.get("redis_db")) + extra: Final = config.get("cache_kwargs") + if isinstance(extra, dict): + settings.update({str(key): value for key, value in extra.items()}) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # the cache_kwargs argument is untyped + nodes: object = settings.get("startup_nodes") # rebind-ok: the env cluster nodes when the arguments name none + if not nodes: + env_nodes: Final[object] = get_secret("REDIS_CLUSTER_NODES") # pyright: ignore[reportUnknownVariableType] # get_secret is untyped + if isinstance(env_nodes, str): + parsed: Final[object] = json.loads(env_nodes) # pyright: ignore[reportAny] # plain JSON + nodes = parsed # rebind-ok: see above + if nodes: + return RedisClusterCache(**{**settings, "startup_nodes": nodes}) # pyright: ignore[reportArgumentType] # RedisCache's keyword arguments + return RedisCache(**settings) # pyright: ignore[reportArgumentType] # RedisCache's keyword arguments + python_host: Final = PythonHost() + + +def _dotted(op: str) -> object: + """``op`` named by its path under ``litellm`` (``vector_stores.main.asearch``), read at call time. + + Each segment is a module attribute, a submodule, or a key of a mapping such as a module's + table of generated endpoints. + """ + node: object = litellm # rebind-ok: walks the path one segment at a time + path: str = "litellm" # rebind-ok: the module path walked so far + for segment in op.split("."): + path = f"{path}.{segment}" + if isinstance(node, Mapping): + node = cast(Mapping[str, object], node).get(segment) # cast-ok: endpoint tables are keyed by name + elif isinstance(node, types.ModuleType) and (submodule := _submodule(path)) is not None: + node = submodule + else: + node = cast(object, getattr(node, segment, None)) # cast-ok: any attribute + if node is None: + return None + return node + + +def _submodule(path: str) -> types.ModuleType | None: + try: + return importlib.import_module(path) + except ModuleNotFoundError: + return None diff --git a/litellm/router_rust/__init__.py b/litellm/router_rust/__init__.py new file mode 100644 index 00000000000..42cfc56aca4 --- /dev/null +++ b/litellm/router_rust/__init__.py @@ -0,0 +1,13 @@ +""" +The Rust router core's Python side: ``RustCore`` (``core``) and the host-side modules router-core +(litellm-rust/crates/router-core) reaches through its steps. + +- ``steps``: answers to host steps that need live Python objects (strategies, pre-routing, + pre-call facts, plugins, callbacks, caller access) +- ``invoke``: the host side of ``Host::invoke``, one attempt against one deployment (request + shaping, endpoints, streams, slots, retry and fallback metadata, prompt management, websockets) +- ``surface``: the Python ``Router`` API kept over router-core's state (registry views, model + info, settings, health, retries, fallbacks, alerting); a Rust host needs none of it + +Nothing is imported here, so importing one module does not import the core. +""" diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py new file mode 100644 index 00000000000..395b580d9b2 --- /dev/null +++ b/litellm/router_rust/core.py @@ -0,0 +1,4943 @@ +""" +``RustCore``: the router core served by litellm-rust/crates/router-core through ``_native``. + +The Rust core owns the registry, selection, retries, fallbacks and cooldowns. It reaches outside +itself only through its host, and the host never calls back into a Python router: + +- every attempt runs through the host: ``router_attempt`` builds the request for the one + deployment Rust picked and ``PythonHost.invoke(op)`` calls ``litellm.`` with it +- the core records every outcome (cooldowns, allowed-fails counters, usage) itself, in the routing + cache the host builds and owns (``PythonHost.routing_cache``), as far as the outcome callbacks the + Python router would have running for the request reach (``OptionalChecks.outcome_callbacks``) +- sleeps, fallback access and budget checks, and cache reads and writes are host steps + +``RustCore`` does not build, wrap or read a ``PythonCore``. Settings are resolved from the +``RouterConfig`` and the host's process settings; the views (``registry()``, ``settings()``, +``health()``) are read from the Rust handle. A member the Rust core does not serve raises +``RustCoreUnsupported`` (an ``AttributeError``) instead of reaching a Python core, so deleting +``router_python_core.py`` cannot change what this core does. +""" + +import contextlib +import dataclasses +import inspect +import itertools +import json +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Mapping, Sequence +from datetime import date, datetime, time +from functools import partial +from time import perf_counter +from time import time as unix_time +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, final + +from litellm._internal_context import service_target +from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, HEALTH_CHECKS_TARGET +from litellm.integrations.custom_guardrail import is_guardrail_intervention +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + add_fallback_headers_to_response, + add_retry_headers_to_response, + apply_quality_router_decision_headers, + apply_remaining_usage_headers, + apply_response_model_id, + complexity_router_decision_headers, + ensure_response_additional_headers, + prepare_response_for_header_attachment, +) +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, without +from litellm.litellm_core_utils.routing_common_utils import ( + format_content_policy_fallback_hint, + format_fallback_outcome_message, + format_no_fallback_group_message, + get_request_team_id, + is_web_search_request, + truncate_fallback_error_detail, +) +from litellm.litellm_core_utils.routing_groups import validate_routing_strategy +from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.router_api import ( + ROUTER_INIT_ARGS, + SETTING_FIELDS, + ApplyResult, + ChangeSet, + RouterHealth, + RouterRegistry, + RouterSettings, + RoutingContext, + RoutingObservers, +) +from litellm.router_attempt import for_requested_model, request_from_call +from litellm.router_config import RouterConfig, RouterDynamicConfig, detached +from litellm.router_python_host import PythonHost +from litellm.router_rust.invoke.endpoints import ( + AttemptCall, + Plan, + RustEndpoints, + arm_compaction_state, + attempt_call, + clean_realtime_session_deployment, + compacted_attempt, + failover_fields, + generic_plan, + log_refused_call, + log_refused_call_sync, + pinned_video_deployment, + plan_for, + reports_own_success, + scoped_fallback_fields, + selection_endpoint, + stamp_error, + unrouted_fields, +) +from litellm.router_rust.invoke.shaping import Shaping +from litellm.router_rust.invoke.slots import enter_deployment_slot, max_parallel_requests_limit +from litellm.router_rust.invoke.streaming import ( + StreamAttempt, + continuation_flag, + continuation_routing, + open_attempt_stream, + open_attempt_stream_sync, + register_continuation_prefill_check, + shape_attempt_kwargs, +) +from litellm.router_rust.invoke.walk import FallbackOutcomes, WalkMetadata, await_all, log_retry_into +from litellm.router_rust.steps.access import caller_fields, selection_refusal +from litellm.router_rust.steps.callbacks import OptionalChecks, RustOptionalChecks +from litellm.router_rust.steps.plugins import SYNC_PATH_MESSAGE, candidate_models, run_plugins +from litellm.router_rust.steps.precall import ( + DECISIONS_MEDIA_CALL_TYPES, + PreCallFacts, + cannot_serve_route, + declares_no_vision, + refuses_decisions_media, + request_carries_image, + serves_route_capability, +) +from litellm.router_rust.steps.prerouting import PreRouter, PreRoutingStrategies, picks_async +from litellm.router_rust.steps.selection import ( + SELECTOR_ATTRIBUTES, + StrategySelectors, + group_documents, + group_settings, + member_map, + normalize_strategy, + parse_groups, + pick_async, + pick_sync, + picked_id, + routing_pre_call_checks, +) +from litellm.router_rust.surface.alerting import initialize_alerting +from litellm.router_rust.surface.fallbacks import ( + RustFallbacks, + mock_error, + mock_failure, + refuse_malformed_fallbacks, + rejected_fallback_target, +) +from litellm.router_rust.surface.health import ( + RustHealthCallbacks, + failure_already_logged, + failure_status, + first_cooldown, + logged_call, +) +from litellm.router_rust.surface.health_routing import ( + HEALTH_STATE_KEY, + HealthRouting, + failover_document, + fully_unhealthy_model_names, + resolved_allowed_fails_policy, +) +from litellm.router_rust.surface.health_routing import all_blocked as deployments_all_blocked +from litellm.router_rust.surface.health_routing import fully_blocked_model_names as blocked_model_names +from litellm.router_rust.surface.health_routing import is_blocked as deployment_is_blocked +from litellm.router_rust.surface.legacy_members import RustLegacyMembers +from litellm.router_rust.surface.members import RustRegistryMembers +from litellm.router_rust.surface.model_list import typed_deployment +from litellm.router_rust.surface.registry import ( + DEFAULT_DEPLOYMENT_COPIED_ID, + DEFAULT_DEPLOYMENT_FAILING_ID, + INDEX_ATTRIBUTES, + BuildInstall, + HostRegistry, + RustRegistry, + default_deployment_call_error, + default_deployment_copy, + default_deployment_copy_error, + hosted_registry_view, + index_document, + served_default_deployment, +) +from litellm.router_rust.surface.replay import track_live_registry, untrack_live_registry +from litellm.router_rust.surface.retries import ( + GroupPolicies, + RustRetries, + group_policies, + is_policy_per_group, + policy_lookup_error, + raising_groups, + readable_retry_policy, + reads_as_policy, + request_retry_fields, + resolved_retry_policy, + retry_count_error, + static_group_policies, +) +from litellm.router_rust.surface.settings import ( + MISSING, + PLAIN_DEFAULTS, + SettingsUpdate, + apply_construction_effects, + argument_attribute, + default_litellm_params, + initial_plain, + read_settings_update, + request_timeout, + settings_view, +) +from litellm.router_strategy.least_busy import CALL_IDENTITY, call_id_of, owning_picks, pick_reserves, picks_reserve +from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET +from litellm.router_utils.cooldown_handlers import get_cooldown_deployment_info, mark_deployment_selection_failure +from litellm.router_utils.endpoint_eligibility import endpoint_for_call, selected_endpoint +from litellm.router_utils.kubernetes_pod_discovery import ( + KubernetesPodDiscovery, + async_resolve_pods_after_bound, + resolve_pods_after_bound, +) +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingReadBatch +from litellm.types.router import ( + Deployment, + RouterErrors, + RouterGeneralSettings, + RouterRateLimitError, + RoutingGroup, + RoutingStrategy, +) +from litellm.types.utils import LlmProviders + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + from litellm.caching.dual_cache import DualCache + from litellm.caching.redis_cache import RedisCache + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.router_api import UntypedDict + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.router_utils.health_state_cache import DeploymentHealthCache + from litellm.rust_bridge._native import ( + RouterCall, + RouterCoreHandle, + _CallSteps, # pyright: ignore[reportPrivateUsage] # the stub's step shapes + _FailureReportSteps, # pyright: ignore[reportPrivateUsage] # see above + _SuccessReportSteps, # pyright: ignore[reportPrivateUsage] # see above + ) + from litellm.types.router import ( + CustomRoutingStrategyBase, + DeploymentTypedDict, + GuardrailTypedDict, + PreRoutingHookResponse, + RoutingPlugin, + ) + +_FAILURE_KINDS: Final = ( + "RateLimitError", + "ContextWindowExceededError", + "ContentPolicyViolationError", + "AuthenticationError", + "PermissionDeniedError", + "NotFoundError", + "BadRequestError", + "Timeout", + "APIConnectionError", + "InternalServerError", + "ServiceUnavailableError", + "BadGatewayError", + "GuardrailIntervention", + "CallerCredentialAuthenticationError", + "CallerCredentialRateLimitError", +) +_DECLINED: Final = frozenset({"MalformedRequestFieldError", "UnsupportedEndpointError"}) +_PROVIDERS: Final = tuple(provider.value for provider in LlmProviders) +_DEFAULT_COOLDOWN_SECONDS: Final = 5.0 +# Past this a retry-after no longer fits the Rust core's durations; it cools down as good as forever. +_LONGEST_RETRY_AFTER_SECONDS: Final = 2**63 + +# Constructor arguments this core reads. Any other argument set to something other than its +# unset value keeps the router on the Python core while both exist (make_core decides that from +# the config alone, before any core is built). +_SERVED_ARGUMENTS: Final = frozenset( + { + "model_list", + "routing_strategy", + "routing_strategy_args", + "routing_groups", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "model_group_alias", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_after", + "timeout", + "default_litellm_params", + "fallback_access_check", + "fallback_budget_check", + "enable_mid_stream_fallback_continuation", + "redis_url", + "redis_host", + "redis_port", + "redis_password", + "redis_db", + "cache_kwargs", + "set_verbose", + "debug_level", + # Read host-side by the endpoint plans (litellm/router_rust/invoke/endpoints.py). + "stream_timeout", + "cache_responses", + "assistants_config", + "search_tools", + "guardrail_list", + "default_priority", + "polling_interval", + "ignore_invalid_deployments", + # Read host-side by the pre-routing strategies (litellm/router_rust/steps/prerouting.py). + "auto_router_capability_limit", + # Decided in router-core (precall.rs); its facts come from litellm/router_rust/steps/precall.py. + "enable_pre_call_checks", + # Run host-side (litellm/router_rust/steps/plugins.py); router-core filters on what they leave. + "plugins", + "retry_policy", + "model_group_retry_policy", + # Health-check routing and the failover flags (litellm/router_rust/surface/health_routing.py). + "enable_health_check_routing", + "health_check_staleness_threshold", + "health_check_ignore_transient_errors", + "background_health_check_model_groups", + "enable_weighted_failover", + "fallback_on_unsupported_params", + # Built host-side as CustomLogger checks (litellm/router_rust/steps/callbacks.py); router-core runs + # their deployment filters through its callback_filters stage. + "optional_pre_call_checks", + "provider_budget_config", + "deployment_affinity_ttl_seconds", + "session_affinity_repin_on_failover", + "model_group_affinity_config", + # Kept as given (litellm/router_rust/surface/settings.py) and read host-side by the endpoint plans + # (default_litellm_params' caching groups) and the slot limits. + "client_ttl", + "caching_groups", + "default_max_parallel_requests", + # Built host-side into a Slack alerting logger (litellm/router_rust/surface/alerting.py). + "alerting_config", + # Kept as given and reported by settings(); nothing in the router reads its fields. + "router_general_settings", + } +) +# Arguments that may be passed at their default without asking for anything. +_UNSET_DEFAULTS: Final[Mapping[str, object]] = MappingProxyType( + { + "cache_responses": False, + "enable_pre_call_checks": False, + "enable_tag_filtering": False, + "retry_policy": None, + "model_group_retry_policy": {}, + } +) +# Settings a changeset may change. Static ones rebuild the Rust core (settings are fixed for a +# core's life); dynamic ones are handed to it as a new dynamic document. +_STATIC_SETTINGS: Final = frozenset( + { + "routing_strategy", + "routing_groups", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_after", + "timeout", + "enable_pre_call_checks", + } +) +_DYNAMIC_SETTINGS: Final = frozenset( + { + "default_deployment", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "model_group_alias", + "default_litellm_params", + "fallback_access_check", + "fallback_budget_check", + "routing_strategy_args", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + } +) +# Settings only the host reads (the endpoint plans, the slot limits, the budget limiter, the +# settings view). A changeset assigns them as given, the Rust core never sees them, and +# `update_settings` ignores them (none is runtime-updatable). +_HOST_SETTINGS: Final = frozenset( + { + "cache_responses", + "search_tools", + "guardrail_list", + "default_max_parallel_requests", + "provider_budget_config", + "health_check_ignore_transient_errors", + "router_general_settings", + "provider_default_deployment_ids", + "kubernetes_pod_discovery", + # Read by router-core's health-check filter: assigning one rebuilds the Rust core. + "enable_health_check_routing", + "background_health_check_model_groups", + } +) +# The constructor's default `router_general_settings`: one object every router built without the +# argument shares, as the Python core's default argument is. +_DEFAULT_GENERAL_SETTINGS: Final = RouterGeneralSettings() + + +# Plain settings `update_settings` sets that the Rust core's retry loop reads. +_RETRY_POLICY_SETTINGS: Final = ("retry_policy", "model_group_retry_policy") +# Attributes a `del` unsets beyond the setting fields, whose readers then raise AttributeError. +_DELETABLE_ATTRIBUTES: Final = frozenset({"model_group_alias", "num_retries", "scheduler"}) + + +# Calls whose pick least-busy does not hold (`picks_reserve(False)`): a batch cancel makes no +# call that logs a start. +_UNRESERVED_OPS: Final = frozenset({"acancel_batch"}) + + +# Settings converted by `Resolved._selection_setting`. +_SELECTION_SETTINGS: Final = frozenset( + { + "routing_strategy", + "routing_strategy_args", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + } +) +_NUMBER_SETTINGS: Final = frozenset( + {"num_retries", "max_fallbacks", "allowed_fails", "cooldown_time", "retry_after", "timeout"} +) +# The settings `_core_update_settings` casts with `int(value)`. +_UPDATE_INT_SETTINGS: Final = frozenset({"timeout", "num_retries", "retry_after", "allowed_fails", "cooldown_time"}) + + +class RustCoreUnsupported(AttributeError): + """A member or request the Rust core does not serve. It is never forwarded to a Python core.""" + + +def _cache_target(key: str) -> str: + """The service target a routing cache read or write is attributed to, as the Python core's.""" + if key == HEALTH_STATE_KEY: + return HEALTH_CHECKS_TARGET + return ROUTER_COOLDOWNS_TARGET if key.endswith(":cooldown") else ROUTER_USAGE_TARGET + + +# --- failures ----------------------------------------------------------------------------------- + + +def failure_kind(error: BaseException) -> str: + names: Final = tuple(cls.__name__ for cls in type(error).__mro__) + if not _DECLINED.isdisjoint(names): + return "Declined" + if is_guardrail_intervention(error): # pyright: ignore[reportArgumentType] # every attempt error is an Exception + return "GuardrailIntervention" + return next((name for name in names if name in _FAILURE_KINDS), "APIError") + + +def retry_after_seconds(error: BaseException) -> float | None: + """The header cooldown of ``deployment_callback_on_failure``: ``retry-after`` alone, as a whole + number or a date, with a negative or unreadable one read as none.""" + from litellm.litellm_core_utils.exception_mapping_utils import ( + _get_response_headers, # pyright: ignore[reportPrivateUsage] # the failure callback's own read + ) + from litellm.utils import _get_retry_after_from_exception_header # pyright: ignore[reportPrivateUsage] # see above + + headers: Final = _get_response_headers(original_exception=error) # pyright: ignore[reportArgumentType] # every attempt error is an Exception + if headers is None: + return None + seconds: Final = _get_retry_after_from_exception_header(response_headers=headers) # pyright: ignore[reportUnknownVariableType] # an untyped helper's int or None + return float(min(seconds, _LONGEST_RETRY_AFTER_SECONDS)) if seconds is not None and seconds >= 0 else None + + +def attempt_failure(error: BaseException) -> Mapping[str, object]: + status: Final = getattr(error, "status_code", None) + num_retries: Final = getattr(error, "num_retries", None) + # `_deployment_ids_to_skip_on_retry` reads the id off the exception. + named_deployment: Final = getattr(error, "retry_skip_deployment_id", None) or getattr( + error, "failed_deployment_id", None + ) + return { + "ok": False, + "kind": failure_kind(error), + "status": status if isinstance(status, int) and 100 <= status <= 599 else None, + "status_absent": getattr(error, "status_code", "") == "", + "retry_after": retry_after_seconds(error), + "num_retries": num_retries if isinstance(num_retries, int) and num_retries >= 0 else None, + "failed_deployment_id": named_deployment if isinstance(named_deployment, str) else None, + "detail": str(error), + "classes": [cls.__name__ for cls in type(error).__mro__], + "llm_provider": bool(getattr(error, "llm_provider", "")), + "agentic_followup": is_agentic_followup_failure(error), + } + + +def total_tokens(response: object) -> int: + usage: Final = getattr(response, "usage", None) + tokens: Final = getattr(usage, "total_tokens", None) + return tokens if isinstance(tokens, int) else 0 + + +# --- settings ----------------------------------------------------------------------------------- + + +def _json_default(value: object) -> object: + # ISO 8601, which ``typed_deployment`` reads back as the value it was. + if isinstance(value, date | time): + return value.isoformat() + dump: Final = getattr(value, "model_dump", None) + return dump(exclude_none=True) if callable(dump) else str(value) + + +def _plain(value: object) -> object: + """``value`` as plain JSON values, the shape the Rust core reads.""" + loaded: Final[object] = json.loads(json.dumps(value, default=_json_default)) # pyright: ignore[reportAny] # plain JSON + return loaded + + +def _document(value: object) -> Mapping[str, object]: + """A mapping as the plain JSON document the Rust core reads.""" + return as_str_mapping(_plain(value)) or {} + + +def _as_list(value: object) -> Sequence[object]: + return value if isinstance(value, list) else () # pyright: ignore[reportUnknownVariableType] # a list's items are read as objects + + +def _strategy_name(value: object) -> str | None: + """A strategy as a setting stores it: ``None`` stays no strategy (``_normalize_strategy``).""" + return None if value is None else normalize_strategy(value) or "simple-shuffle" + + +def _lenient_int(value: object, default: int) -> int: + """A count setting kept as given, as the crate reads it: an int, or a number spelled another way + (``1.5``, ``"3"``) truncated to one; ``default`` when it is no number. The Python core stores the + value as given and never checks it at construction.""" + if isinstance(value, bool | int): + return int(value) + try: + return int(float(str(value))) + except (ValueError, OverflowError): + return default + + +def _lenient_float(value: object, default: float | None) -> float | None: + """A seconds setting kept as given, as the crate reads it (see ``_lenient_int``).""" + if isinstance(value, bool | int | float): + return float(value) + try: + return float(str(value)) + except (ValueError, OverflowError): + return default + + +# Settings the Python core stores as given, so that a read returns the value passed (``"1"``, +# ``None``, ``[1]``); ``Resolved`` keeps them in ``given`` and the typed fields hold what the crate reads. +_KEPT_AS_GIVEN: Final = frozenset( + { + "num_retries", + "max_fallbacks", + "allowed_fails", + "retry_after", + "timeout", + "disable_cooldowns", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "routing_strategy_args", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + } +) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Resolved: + """The router's settings, resolved from its arguments and the process settings the way the + Python router's constructor resolves them. Replaced, never changed, by ``apply``.""" + + routing_strategy: str | None + routing_strategy_args: Mapping[str, object] + num_retries: int + max_fallbacks: int + allowed_fails: int + explicit_allowed_fails: int | None + cooldown_time: float + disable_cooldowns: bool | None + allowed_fails_policy: object + retry_after: float + timeout: float | None + fallbacks: list[object] | None # mutable-ok: the setting as Python spells it + context_window_fallbacks: list[object] | None # mutable-ok: same + content_policy_fallbacks: list[object] | None # mutable-ok: same + model_group_alias: Mapping[str, object] + default_litellm_params: Mapping[str, object] + fallback_access_check: object + fallback_budget_check: object + enable_mid_stream_fallback_continuation: bool = False + routing_groups: "tuple[RoutingGroup, ...]" = () + enable_tag_filtering: bool = False + tag_filtering_match_any: bool = True + tag_routing_prefix: str = "" + enable_pre_call_checks: bool = False + routing_plugins: "tuple[RoutingPlugin, ...]" = () + retry_policy: object = None + model_group_retry_policy: object = None + health_routing: HealthRouting = dataclasses.field(default_factory=HealthRouting) + # `_core_default_deployment`, as assigned (detached). + default_deployment: object = None + # `_model_to_group` as the Python core last built it, kept once a raw `routing_groups` write + # replaced the groups without rebuilding it. + routing_group_members: Mapping[str, str] | None = None + # `_routing_groups_input`: the groups last passed to the constructor or `update_settings`, which + # a strategy change through `update_settings` re-inits the routing groups from. + routing_groups_input: object = None + assigned: frozenset[str] = frozenset() + # The `_KEPT_AS_GIVEN` settings as the Python core stores them, for reads. + given: Mapping[str, object] = dataclasses.field(default_factory=lambda: MappingProxyType({})) + + def read(self, name: str) -> object: + """A setting as the Python core reads it back: as given when it keeps it so.""" + return self.given[name] if name in self.given else getattr(self, name) # pyright: ignore[reportAny] # a setting field + + @classmethod + def of(cls, config: RouterConfig, dynamic: RouterDynamicConfig) -> "Resolved": + get: Final = config.get + # The process list is the router's own, as in the Python router: default_fallbacks are + # appended to it in place. + configured: Final[list[object] | None] = ( # mutable-ok: the router's live list, as Python keeps it + list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given + ) + refuse_malformed_fallbacks(configured) + default_fallbacks: Final = get("default_fallbacks") or dynamic.default_fallbacks + if isinstance(default_fallbacks, str): + raise TypeError( + f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']" + ) + if default_fallbacks is not None and configured is not None: + configured.append({"*": default_fallbacks}) + fallbacks: Final[list[object] | None] = ( # mutable-ok: see above + [{"*": default_fallbacks}] if default_fallbacks is not None and configured is None else configured + ) + content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or ( + list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None + ) + refuse_malformed_fallbacks(content_policy) + num_retries: Final = get("num_retries") + max_fallbacks: Final = get("max_fallbacks") + allowed_fails: Final = get("allowed_fails") + timeout: Final = get("timeout") + strategy: Final = get("routing_strategy") + validate_routing_strategy(strategy) # pyright: ignore[reportArgumentType] # validated as given + # The values the Python core's constructor stores (`self.num_retries = num_retries`, ...). + given_num_retries: Final[object] = ( + num_retries if num_retries is not None else dynamic.num_retries if dynamic.num_retries is not None else 2 + ) + given_max_fallbacks: Final[object] = ( + max_fallbacks + if max_fallbacks is not None + else dynamic.max_fallbacks + if dynamic.max_fallbacks is not None + else dynamic.router_max_fallbacks + ) + given_allowed_fails: Final[object] = allowed_fails if allowed_fails is not None else dynamic.allowed_fails + given_timeout: Final[object] = timeout or dynamic.request_timeout + given_retry_after: Final[object] = get("retry_after") if "retry_after" in config.arguments else 0 + kept: Final = { + "num_retries": given_num_retries, + "max_fallbacks": given_max_fallbacks, + "allowed_fails": given_allowed_fails, + "timeout": given_timeout, + "retry_after": given_retry_after, + "disable_cooldowns": get("disable_cooldowns"), + **{ + name: get(name) + for name in ("enable_tag_filtering", "tag_filtering_match_any", "tag_routing_prefix") + if name in config.arguments + }, + **( + {"routing_strategy_args": get("routing_strategy_args")} + if "routing_strategy_args" in config.arguments + else {} + ), + } + return cls( + # An explicit `routing_strategy=None` is stored as None (no strategy picks). + routing_strategy=None + if strategy is None and "routing_strategy" in config.arguments + else normalize_strategy(strategy) or "simple-shuffle", + # A plain dict, as `self.routing_strategy_args`: `/router/settings` serializes it as read. + routing_strategy_args=dict(as_str_mapping(get("routing_strategy_args")) or {}), + num_retries=_lenient_int(given_num_retries, 2), + max_fallbacks=_lenient_int(given_max_fallbacks, dynamic.router_max_fallbacks), + allowed_fails=_lenient_int(given_allowed_fails, dynamic.allowed_fails), + explicit_allowed_fails=_lenient_int(allowed_fails, dynamic.allowed_fails) + if allowed_fails is not None + else None, + cooldown_time=_lenient_float(get("cooldown_time") or _DEFAULT_COOLDOWN_SECONDS, _DEFAULT_COOLDOWN_SECONDS) + or 0.0, + disable_cooldowns=_optional_bool(get("disable_cooldowns")), + allowed_fails_policy=resolved_allowed_fails_policy(get("allowed_fails_policy")), + retry_after=_lenient_float(given_retry_after or 0, 0.0) or 0.0, + timeout=_lenient_float(given_timeout, None) if given_timeout else None, + given=MappingProxyType(kept), + fallbacks=fallbacks, + context_window_fallbacks=list(_as_list(get("context_window_fallbacks"))) + or (list(dynamic.context_window_fallbacks) if dynamic.context_window_fallbacks else None), + content_policy_fallbacks=content_policy, + model_group_alias=MappingProxyType(dict(as_str_mapping(get("model_group_alias")) or {})), + default_litellm_params=MappingProxyType(dict(as_str_mapping(get("default_litellm_params")) or {})), + fallback_access_check=get("fallback_access_check"), + fallback_budget_check=get("fallback_budget_check"), + enable_mid_stream_fallback_continuation=continuation_flag(get("enable_mid_stream_fallback_continuation")), + enable_tag_filtering=bool(get("enable_tag_filtering")), + tag_filtering_match_any=_true_unless_given(get("tag_filtering_match_any")), + tag_routing_prefix=str(get("tag_routing_prefix") or ""), + enable_pre_call_checks=bool(get("enable_pre_call_checks")), + routing_plugins=tuple(_as_list(get("plugins"))), # pyright: ignore[reportArgumentType] # the plugins as passed + routing_groups_input=get("routing_groups"), + routing_groups=parse_groups( + get("routing_groups"), + frozenset( + str(name) + for entry in _as_list(get("model_list")) + if (name := (as_str_mapping(entry) or {}).get("model_name")) + ), + frozenset(as_str_mapping(get("model_group_alias")) or {}), + ), + retry_policy=resolved_retry_policy(get("retry_policy")), + model_group_retry_policy=get("model_group_retry_policy"), + health_routing=HealthRouting.of(config), + ) + + def changed(self, values: Mapping[str, object], updated: frozenset[str] = frozenset()) -> "Resolved": + """``values`` applied; ``updated`` names those passed as ``ChangeSet.settings`` (which go + through ``update_settings``: only they may name ``routing_strategy_args``).""" + from litellm.router_api import ASSIGNABLE_ALIASES, PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS + + unknown: Final = sorted( + set(values) + - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES} + - (updated & {"routing_strategy_args", "routing_groups", "enable_tag_filtering", "tag_routing_prefix"}) + ) + if unknown: + raise ValueError(f"not assignable router settings: {unknown}") + unserved: Final = sorted(set(values) - _STATIC_SETTINGS - _DYNAMIC_SETTINGS) + if unserved: + raise RustCoreUnsupported(f"the Rust router core does not serve the setting(s) {', '.join(unserved)} yet") + updates: Final[dict[str, object]] = {} # mutable-ok: built here, then applied once + for name, value in values.items(): + if name in {"model_group_alias", "default_litellm_params"}: + updates[name] = MappingProxyType(dict(as_str_mapping(value) or {})) + elif name in _NUMBER_SETTINGS: + updates[name] = self._number_setting(name, value) + if name == "allowed_fails": + updates["explicit_allowed_fails"] = updates[name] + elif name in _SELECTION_SETTINGS: + updates[name] = self._selection_setting(name, value) + elif name == "disable_cooldowns": + updates[name] = _optional_bool(value) + elif name == "enable_pre_call_checks": + updates[name] = bool(value) + elif name in ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks"): + updates[name] = list(_as_list(value)) or None + elif name == "default_deployment": + updates[name] = detached(value) + else: + updates[name] = value + if "routing_groups" in values: + # `_init_routing_groups` rebuilds the member map from the new groups. + updates["routing_group_members"] = None + # `_core_update_settings` casts its int settings (`int(value)`, which raises as Python's + # does); every other write is stored as given. + kept: Final = { + name: int(value) if name in updated and name in _UPDATE_INT_SETTINGS else value # pyright: ignore[reportArgumentType] # int() of the value as given, as Python casts it + for name, value in values.items() + if name in _KEPT_AS_GIVEN + } + return dataclasses.replace( + self, + **updates, + assigned=self.assigned | frozenset(values), + given=MappingProxyType({**self.given, **kept}), + ) # pyright: ignore[reportArgumentType] # each value converted to its field's type above + + def _number_setting(self, name: str, value: object) -> object: + """A numeric setting as the crate reads it; the value as given is kept in ``given``.""" + if name in ("num_retries", "max_fallbacks", "allowed_fails"): + return _lenient_int(value, getattr(self, name)) # pyright: ignore[reportAny] # the field's current int + if name == "cooldown_time": + # Stored as given (`_core_cooldown_time`), so a failure cools down for that long: + # 0 for no time at all; None leaves the cooldown cache's own default, set by the caller. + return _lenient_float(value, self.cooldown_time) if value is not None else self.cooldown_time + if name == "retry_after": + return _lenient_float(value or 0, self.retry_after) + return _lenient_float(value, self.timeout) if value is not None else None + + def _selection_setting(self, name: str, value: object) -> object: + """A selection setting converted to its field's type.""" + if name == "routing_strategy": + return _strategy_name(value) + if name == "routing_strategy_args": + return dict(as_str_mapping(value) or {}) + if name == "tag_routing_prefix": + return str(value or "") + return bool(value) + + def static_document(self, model_list: Sequence[object]) -> Mapping[str, object]: + values: Final = { + "model_list": _plain(list(model_list)), + "routing_strategy": self.routing_strategy, + "cooldown_time": self.cooldown_time, + "disable_cooldowns": self.disable_cooldowns, + "allowed_fails_policy": _plain(self.allowed_fails_policy), + "retry_after": self.retry_after, + "timeout": self.timeout, + "routing_groups": group_documents(self.routing_groups) if self.routing_groups else None, + "enable_pre_call_checks": True if self.enable_pre_call_checks else None, + "routing_plugins": True if self.routing_plugins else None, + # `async_callback_filter_deployments` runs on every async candidate list. + "callback_filters": True, + "retry_policy": _plain(self.retry_policy), + "model_group_retry_policy": _plain(static_group_policies(self.model_group_retry_policy)) or None, + "routing_group_members": dict(self.routing_group_members) + if self.routing_group_members is not None + else None, + **self.health_routing.static_document(), + } + kept: Final = {name: value for name, value in values.items() if value is not None} + # An explicit `routing_strategy=None` reaches the Rust core as "None": no strategy picks. + return kept if self.routing_strategy is not None else {**kept, "routing_strategy": "None"} + + def dynamic_document(self, version: int, model_names: Sequence[str]) -> Mapping[str, object]: + values: Final = { + "version": version, + "model_group_alias": _plain(dict(self.model_group_alias)), + "fallbacks": _plain(self.fallbacks), + "context_window_fallbacks": _plain(self.context_window_fallbacks), + "content_policy_fallbacks": _plain(self.content_policy_fallbacks), + "num_retries": self.num_retries, + "max_fallbacks": self.max_fallbacks, + "allowed_fails": self.allowed_fails, + "allowed_fails_is_default": True if self.allowed_fails_is_default() else None, + "fallback_keys": self._fallback_keys(model_names), + "default_deployment": _plain(served_default_deployment(self.default_deployment)[0]), + "default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None, + } + return {name: value for name, value in values.items() if value is not None} + + def allowed_fails_is_default(self) -> bool: + """``allowed_fails`` does not count as set on the router (``_is_allowed_fails_set_on_router``): + not passed explicitly, and the value copied at init still equals the live process default.""" + import litellm + + return self.explicit_allowed_fails is None and self.allowed_fails == litellm.allowed_fails + + def _fallback_keys(self, model_names: Sequence[str]) -> Mapping[str, object]: + """The spellings a fallback key matches besides the group name (`get_fallback_model_group`).""" + from litellm.router_utils.fallback_event_handlers import ( + _provider_prefixed_model_group, # pyright: ignore[reportPrivateUsage] # the fallback walk's own spelling rule + ) + + keys: Final = tuple( # comprehension-ok: the first group of each fallback entry, across the three lists + str(next(iter(mapping))) + for entries in (self.fallbacks, self.context_window_fallbacks, self.content_policy_fallbacks) + for entry in _as_list(entries) + if (mapping := as_str_mapping(entry)) + ) + names: Final = {*model_names, *self.model_group_alias} + prefixed: Final = { + name: spelled for name in names if (spelled := _provider_prefixed_model_group(name, keys)) is not None + } + from litellm.types.router import LiteLLMParamsTypedDict + + return { + "providers": _PROVIDERS, + "provider_prefixed": prefixed, + "param_keys": list(LiteLLMParamsTypedDict.__annotations__), + } + + +def _alias_target(target: object) -> str | None: + mapping: Final = as_str_mapping(target) + if mapping is not None: + model: Final = mapping.get("model") + return model if isinstance(model, str) else None + return target if isinstance(target, str) else None + + +def _true_unless_given(value: object) -> bool: + return True if value is None else bool(value) + + +def _group_selectors(groups: "Sequence[RoutingGroup]") -> Sequence[tuple[str, str, Mapping[str, object]]]: + return [ + ( + group.group_name, + group.routing_strategy, + as_str_mapping(group.routing_strategy_args) or {}, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # RoutingGroup's untyped dict + ) + for group in groups + ] + + +def _optional_bool(value: object) -> bool | None: + return None if value is None else bool(value) + + +def unsupported_arguments(config: RouterConfig) -> frozenset[str]: + """Arguments the Rust core does not serve.""" + unserved: Final = frozenset( + name + for name, value in config.arguments.items() + if name not in _SERVED_ARGUMENTS and value not in (None, _UNSET_DEFAULTS.get(name)) + ) + return unserved + + +def _listing_team_id(request: Mapping[str, object]) -> str | None: + """``(kwargs.get("metadata", {}) or {}).get("user_api_key_team_id")``: the team + ``async_function_with_fallbacks_common_utils`` lists a group for reads this bucket alone.""" + metadata: Final = as_str_mapping(request.get("metadata")) + team_id: Final = metadata.get("user_api_key_team_id") if metadata is not None else None + return team_id if isinstance(team_id, str) else None + + +# Request fallback settings, which replace the router's for the call. +_REQUEST_FALLBACK_KEYS: Final = ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks", "max_fallbacks") + + +def _strings(value: object) -> "tuple[str, ...]": + """Every string in ``value``, a name or a fallback list of names and dict entries.""" + if isinstance(value, str): + return (value,) + if isinstance(value, Mapping): + entries: Final[Mapping[object, object]] = value # pyright: ignore[reportUnknownVariableType] # read as objects + return tuple(itertools.chain.from_iterable(map(_strings, itertools.chain.from_iterable(entries.items())))) + if isinstance(value, list | tuple): + items: Final[Sequence[object]] = value # pyright: ignore[reportUnknownVariableType] # see above + return tuple(itertools.chain.from_iterable(map(_strings, items))) # pyright: ignore[reportUnknownArgumentType] # read as objects + return () + + +def _routed_model(request: Mapping[str, object]) -> object: + """The request's ``model`` as a recovery of its stream names it (``RustCore.routed_name``).""" + model: Final = request.get("model") + return model if model is None or isinstance(model, str) else str(model) + + +def _start_fields(request: Mapping[str, object]) -> Mapping[str, object]: + """The request fields the Rust core routes on.""" + fields: Final = { + **request_retry_fields(request.get("num_retries")), + **({"cooldown_time": request["cooldown_time"]} if request.get("cooldown_time") is not None else {}), + } + # `_common_checks_available_deployment`: only True names a litellm model. + specific: Final = {"specific_deployment": True} if request.get("specific_deployment") is True else {} + # `record_disable_fallbacks(kwargs, disable_fallbacks is True)`: only True turns them off. + disabled: Final = {"disable_fallbacks": True} if request.get("disable_fallbacks") is True else {} + lists: Final = {key: _plain(request[key]) for key in _REQUEST_FALLBACK_KEYS if key in request} + weights: Final = request.get("_router_weights") + team_id: Final = get_request_team_id(request) + listing_team_id: Final = _listing_team_id(request) + strategy: Final = request.get("routing_strategy") + override: Final = normalize_strategy(strategy) if isinstance(strategy, str | RoutingStrategy) else None + return { + **fields, + **specific, + **({"routing_strategy": override} if override is not None else {}), + **disabled, + **({"router_weights": weights} if weights is not None else {}), + **lists, + **({"team_id": team_id} if team_id else {}), + **({"listing_team_id": listing_team_id} if listing_team_id is not None else {}), + **caller_fields(request), + } + + +# --- views -------------------------------------------------------------------------------------- + + +class RustSettings(RouterSettings): + """``settings()``: the resolved settings as a frozen snapshot.""" + + def guardrail(self, name: str) -> "GuardrailTypedDict": + router: Final = self._router + if not isinstance(router, _SettingsSource): + return super().guardrail(name) + return router.available_guardrail(name) # pyright: ignore[reportReturnType] # the configured guardrail entry + + def as_dict(self) -> Mapping[str, object]: + router: Final = self._router + if not isinstance(router, _SettingsSource): + return super().as_dict() + return router.settings_dict() + + +class _SettingsSource: + def settings_dict(self) -> Mapping[str, object]: + raise NotImplementedError + + def available_guardrail(self, name: str) -> object: + raise NotImplementedError + + +class RustHealth(RouterHealth): + """``health()``: cooldowns, fail counts and usage, read through the host's routing cache and + written through the Rust core.""" + + __slots__ = ("_core",) + + def __init__(self, core: "RustCore") -> None: # pyright: ignore[reportMissingSuperCall] # reads the Rust core, not a Router + self._core = core + + def __getattribute__(self, name: str) -> object: + if name in _HEALTH_SERVED or name.startswith("__") or name == "_core": + found: Final[object] = object.__getattribute__(self, name) # pyright: ignore[reportAny] # the view's own member + return found + raise RustCoreUnsupported(f"health().{name} is not served by the Rust router core", name=name, obj=self) + + @property + def store(self) -> "DualCache": + return self._core._cache # pyright: ignore[reportPrivateUsage] # the health view reads its own core + + @property + def cooldowns(self) -> "CooldownCache": + return self._core._cooldowns # pyright: ignore[reportPrivateUsage] # see above + + def failures_this_minute(self, deployment_id: str) -> int: + count: Final[object] = self.store.get_cache(f"{deployment_id}:fails", local_only=True) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return int(count) if isinstance(count, int | float) else 0 + + def record_failure(self, deployment_id: str, exception: Exception, exception_status: str | int) -> bool: + self._core._sync_allowed_fails() # pyright: ignore[reportPrivateUsage] # see above + failure: Final = { + **attempt_failure(exception), + "status": failure_status(exception_status), + "caller_scoped": self._core.per_user_deployment(deployment_id), + } + handle: Final = self._core._handle # pyright: ignore[reportPrivateUsage] # see above + call: Final = handle.report_failure(deployment_id, failure, count_only=exception_status == "") + done: Final = self._core._drive_sync_steps(call) # pyright: ignore[reportPrivateUsage] # see above + return self._core.cooled_down(done, deployment_id, exception_status) + + def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": + return self._core._hosted._core_team_ptu_ceiling(team_id, model_group) # pyright: ignore[reportPrivateUsage] # the host registry's ported ceiling + + def split_priority(self, model: str, kwargs: Mapping[str, object]) -> tuple[int | None, Mapping[str, object]]: + """``_core_split_request_priority``, as the endpoints resolve a request's priority.""" + return self._core._endpoints._split_priority(model, kwargs) # pyright: ignore[reportPrivateUsage] # see store + + def priority_header_fields( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: + """``priority_header_update``, as the endpoints read a priority header.""" + return self._core._endpoints.priority_header_update(model=model, kwargs=kwargs, header=header) # pyright: ignore[reportPrivateUsage] # see store + + @property + def health_checks(self) -> "DeploymentHealthCache": + return self._core._health_checks # pyright: ignore[reportPrivateUsage] # see store + + @property + def latency(self) -> "UntypedDict": # the live latency map, as the Python core keeps it + return self._core._hosted._core_deployment_latency_map # pyright: ignore[reportPrivateUsage] # see team_ptu_ceiling + + @staticmethod + def is_blocked(deployment: "Deployment") -> bool: + return deployment_is_blocked(deployment) + + @staticmethod + def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: + return deployments_all_blocked(deployments) + + def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set + return blocked_model_names(self._core._hosted._core_get_model_list() or []) # pyright: ignore[reportPrivateUsage, reportArgumentType] # see team_ptu_ceiling + + async def fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set + core: Final = self._core + return await fully_unhealthy_model_names( + core._hosted._core_get_model_list() or [], # pyright: ignore[reportPrivateUsage, reportArgumentType] # see team_ptu_ceiling + core._health_checks, # pyright: ignore[reportPrivateUsage] # see store + core._resolved.allowed_fails_policy, # pyright: ignore[reportPrivateUsage] # see store + ) + + async def usage( # pyright: ignore[reportIncompatibleMethodOverride] # the same two kinds + self, model_group: str, *, kind: Literal["current", "remaining"] = "current" + ) -> tuple[int | None, int | None] | Mapping[str, int]: + """The group's usage this minute, read from the routing cache over the host registry's + live model list, as ``get_model_group_usage`` / ``get_remaining_model_group_usage`` read it.""" + hosted: Final = self._core._hosted # pyright: ignore[reportPrivateUsage] # see above + if kind == "current": + return await hosted._core_get_model_group_usage(model_group) # pyright: ignore[reportPrivateUsage] # the ported usage read + if kind == "remaining": + return await hosted._core_get_remaining_model_group_usage(model_group) # pyright: ignore[reportPrivateUsage] # see above + raise ValueError(f"unknown usage kind {kind!r}") + + +# Deprecated attribute writes (``router. = ...`` reaches the core as ``_core_``) of +# registry state this core keeps on its host registry, and of routing state it rebinds. +_HOSTED_STATE: Final = frozenset( + { + "_core_model_names", + "_core_deployment_names", + "_core_team_public_model_names", + "_core_model_name_to_deployment_indices", + "_core_team_model_to_deployment_indices", + "_core_cached_deployment_model_info", + "_core_pattern_router", + "_core_team_pattern_routers", + "_core_routing_groups", + "_core_auto_routers", + "_core_complexity_routers", + "_core_quality_routers", + "_core_adaptive_routers", + "_core_routing_plugins", + "_core_deployment_latency_map", + } +) +_REBOUND_STATE: Final = MappingProxyType( + { + "_core_cache": "_cache", + "_core_cooldown_cache": "_cooldowns", + "_core_health_state_cache": "_health_checks", + "_core_slack_alerting_logger": "_slack_alerting_logger", + } +) + + +_HEALTH_SERVED: Final = frozenset( + { + "store", + "cooldowns", + "failures_this_minute", + "record_failure", + "usage", + "team_ptu_ceiling", + "split_priority", + "priority_header_fields", + "health_checks", + "latency", + "is_blocked", + "all_blocked", + "fully_blocked_model_names", + "fully_unhealthy_model_names", + } +) + + +# --- the core ----------------------------------------------------------------------------------- + + +_NO_INPUT: Final = object() +_EndpointImpl: TypeAlias = Callable[..., Awaitable[object]] +# A strategy select step's strategy, selector, keyed name and candidate deployments. +_StrategyInputs: TypeAlias = tuple[str, object, str, list[dict[str, object]]] + + +@final +class _PickCounts(Protocol): + """The router cache as ``track_deployment_pick`` uses it: each deployment id holds its pick count.""" + + def get_cache( + self, key: object, parent_otel_span: "Span | None" = None, local_only: bool = False + ) -> int | None: ... + + def set_cache(self, key: object, value: int, local_only: bool = False, *, ttl: int = ...) -> None: ... + + +class _EndpointPort: + """``RustCore`` as its endpoints see it (``EndpointCore``): each member forwards to the core's + own state or call drivers, read live, since the core replaces its handle and settings. The + port is the core's own adapter, so it is the one place its private members are read from + outside the class; every ``reportPrivateUsage`` ignore below is for that reason.""" + + __slots__ = ("_core",) + + def __init__(self, core: "RustCore") -> None: + self._core: Final = core + + @property + def router(self) -> "RustCore": + return self._core + + @property + def host(self) -> PythonHost: + return self._core._host # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def handle(self) -> "RouterCoreHandle": + return self._core._handle # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def resolved(self) -> "Resolved": + return self._core._resolved # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def shaping(self) -> Shaping: + return self._core._shaping # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def routing_cache(self) -> "DualCache": + return self._core._cache # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def selectors(self) -> StrategySelectors: + return self._core._selectors # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def endpoints(self) -> RustEndpoints: + return self._core._endpoints # pyright: ignore[reportPrivateUsage] # see the class docstring + + async def drive( + self, + plan: Plan, + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + passthrough: Callable[[Mapping[str, object], str], Awaitable[object]] | None = None, + prepared_already: bool = False, + ) -> object: + return await self._core._drive( # pyright: ignore[reportPrivateUsage] # see the class docstring + plan, request, overrides, passthrough=passthrough, prepared_already=prepared_already + ) + + def call_sync( + self, plan: Plan, request: Mapping[str, object], overrides: Mapping[str, object] | None = None + ) -> object: + return self._core._call_sync(plan, request, overrides) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def routing_callbacks(self) -> Iterable[object]: + return self._core._checks.routing_callbacks() # pyright: ignore[reportPrivateUsage] # see the class docstring + + def resolve_alias(self, model: str) -> str | None: + return self._core._hosted._core_get_model_from_alias(model) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def group_deployments(self, model_name: str) -> Sequence[Mapping[str, object]] | None: + return self._core._hosted._core_get_model_list(model_name=model_name) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def routed_name(self, model: object) -> str: + return self._core.routed_name(model) + + def service_logged(self, call_type: str, start_time: float, end_time: float, request: Mapping[str, object]) -> None: + self._core.service_logged(call_type, start_time, end_time, request) + + async def set_response_headers( + self, response: object, model_group: str | None, request_kwargs: Mapping[str, object] + ) -> object: + return await self._core.set_response_headers(response, model_group, request_kwargs) + + def configured_deployment(self, model_id: str) -> "Deployment | None": + return self._core._hosted._core_get_deployment(model_id) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def registry(self) -> RouterRegistry: + return self._core.registry() + + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the RouterCore signature + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + return await self._core.select( + model, request_kwargs, kind=kind, specific_deployment=specific_deployment, messages=messages + ) + + async def acompletion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._core.acompletion(*args, **kwargs) + + +class RustCore( + _SettingsSource, + RustLegacyMembers, + RustRegistryMembers, + RustRetries, + RustFallbacks, + RustHealthCallbacks, + RustOptionalChecks, +): + """A ``RouterCore`` served by the Rust router core over a ``PythonHost``. It holds no Python core.""" + + def __init__(self, config: RouterConfig, host: PythonHost, resolved: Resolved) -> None: + from litellm._service_logger import ServiceLogging + from litellm.router_utils.cooldown_cache import CooldownCache + + self._config: Final = config + self._host: Final = host + self.service_logger_obj: ServiceLogging = ServiceLogging() + apply_construction_effects(config, host) + self._resolved: Resolved = resolved + # `cooldown_time` as the Python core stores it (`cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS`, + # then each value assigned or updated, as given): what `settings()` reads back. + self._cooldown_given: object = config.get("cooldown_time") or DEFAULT_COOLDOWN_TIME_SECONDS + # Settings the Python core keeps as plain attributes, and settings deleted with `del`. + self._plain: Final = initial_plain(config) + self._deleted: frozenset[str] = frozenset() + self._request_timeout: Final = request_timeout(config) + legacy: Final[dict[str, object]] = {"deployment_stats": {}} # mutable-ok: the core's own legacy attributes + self._legacy: Final = legacy + self._health_view: RustHealth | None = None + self._cache: DualCache = host.routing_cache(config) + self._cooldowns: CooldownCache = CooldownCache( + cache=self._cache, + default_cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the constructor's value, as the Python core passes it + ) + self._health_checks: DeploymentHealthCache = resolved.health_routing.health_checks(self._cache) + self._version: int = 1 + self._settings: RustSettings | None = None + self._tag_filtering_models: tuple[int, bool] | None = None + # The registry indexes written through the deprecated attributes (`_core_`), and the + # document router-core last received for them. + self._index_written: frozenset[str] = frozenset() + self._index_sent: Mapping[str, object] | None = None + self._selectors: Final = StrategySelectors(host, self._cache) + self._selectors.init_default(resolved.routing_strategy, resolved.routing_strategy_args) + self._selectors.init_groups(_group_selectors(resolved.routing_groups)) + self._port: Final = _EndpointPort(self) + self._endpoints: Final = RustEndpoints(self._port, config) + # The resolver every picked deployment with `kubernetes_pod_discovery` goes through. + self._pod_discovery: KubernetesPodDiscovery = KubernetesPodDiscovery() + general: Final = config.get("router_general_settings", _DEFAULT_GENERAL_SETTINGS) + self._general_settings: object = general or RouterGeneralSettings() + self._registry_view: RouterRegistry | None = None + self._hosted: Final = HostRegistry( + host, + model_group_alias=dict(resolved.model_group_alias), # pyright: ignore[reportArgumentType] # the configured alias map + ignore_invalid_deployments=bool(config.get("ignore_invalid_deployments")), + on_change=self._sync_registry, + ) + self._hosted._core_cache = self._cache # pyright: ignore[reportPrivateUsage] # usage reads come from the cache the Rust core writes + ctx: Final = RoutingContext(self) # pyright: ignore[reportArgumentType] # the strategies read registry(), settings(), health() and select() + strategies: Final = PreRoutingStrategies( + self._hosted, host, ctx, config.get("auto_router_capability_limit"), self._cache + ) + self._hosted.strategies = strategies + self._prerouter: Final = PreRouter(strategies, self._hosted, host, ctx, self._cache) + # `register_plugin`: a custom strategy that replaces every pick (the host answers the + # Rust core's pre_route step with its deployment). + self._custom_strategy: CustomRoutingStrategyBase | None = None + self._custom_discovery: KubernetesPodDiscovery | None = None + # router-core builds the deployments, asking the host registry about each one: the handle + # starts empty, and is rebuilt over the built list for the settings the list's names feed. + self._handle_default_ids: tuple[str, ...] = () # the provider default ids the handle was built with + self._allowed_fails_default_sent: bool = False # what the handle's dynamic document last said + self._walked_fallbacks: Mapping[str, object] = {} # the fallback lists that document held + self._handle: RouterCoreHandle = self._new_handle([]) + self._hosted.build_runner = self._run_build + self._hosted.missing_attribute = self._no_router_attribute + self._hosted.changes_runner = self._apply_registry_changes + self._hosted.handle_lookup = self._live_handle + self._hosted.build(_as_list(config.get("model_list")) if config.get("model_list") is not None else None) + self._hosted.set_routing_groups(resolved.routing_groups) + track_live_registry(self._hosted) + self._hosted.routable_lookup = self._routable + self._hosted.max_parallel_requests_lookup = self._max_parallel_requests_client + self._hosted._core_routing_plugins = list(resolved.routing_plugins) # pyright: ignore[reportPrivateUsage] # the core fills its own host registry + self._rebuild() + self._join_callback_holders() + self._checks: Final = OptionalChecks(self, host, self._cache, self._hosted, ctx, config.arguments) + self._hosted.checks = self._checks + strategies.ensure_affinity = self._checks.ensure_deployment_affinity_callback + self._checks.build(config.get("optional_pre_call_checks"), config.get("model_list")) + self._slack_alerting_logger: SlackAlerting | None = initialize_alerting(host, config.get("alerting_config")) # pyright: ignore[reportArgumentType] # the argument as passed + if self._slack_alerting_logger is not None: + self._checks.slack_alerting_loggers = (self._slack_alerting_logger,) + if resolved.enable_mid_stream_fallback_continuation: + register_continuation_prefill_check(host) + if resolved.routing_strategy == "lar1": + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy(ctx, resolved.routing_strategy_args) # pyright: ignore[reportArgumentType] # the core's RoutingContext + + @property + def _registry_state(self) -> HostRegistry: + return self._hosted + + @property + def _python_host(self) -> PythonHost: + return self._host + + @property + def _written_retry_policy(self) -> object: + return self._plain["retry_policy"] + + @classmethod + def build(cls, config: RouterConfig, host: PythonHost) -> "RustCore": + from litellm.rust_bridge._native import RustBridgeDeclined + + extra: Final = unsupported_arguments(config) + if extra: + raise RustBridgeDeclined(f"router arguments not served by the Rust core: {sorted(extra)}") + RustEndpoints.prepare_host(config, host) + return cls(config, host, Resolved.of(config, host.dynamic())) + + @property + def enable_mid_stream_fallback_continuation(self) -> bool: + return self._resolved.enable_mid_stream_fallback_continuation + + def _set_continuation(self, enabled: bool) -> None: + """``_set_mid_stream_fallback_continuation``: an enabled flag registers the process-wide prefill check.""" + self._resolved = dataclasses.replace(self._resolved, enable_mid_stream_fallback_continuation=enabled) + self._settings = None + if enabled: + register_continuation_prefill_check(self._host) + + def __getattr__(self, name: str) -> object: + if name in self.__dict__.get("_deleted", ()): + raise self._no_attribute(name) + if name in SELECTOR_ATTRIBUTES.values(): + return self._selectors.attribute(name) + if name == "_core_async_get_available_deployment" and self.__dict__.get("_custom_strategy") is not None: + return self._custom_async_pick() + # Only reached for members this class does not define: the endpoints served through the + # host, else refused. Never forwarded anywhere. + endpoints: Final[object] = self.__dict__.get("_endpoints") + endpoint: Final = endpoints.member(name) if isinstance(endpoints, RustEndpoints) else None + if endpoint is not None: + return endpoint + attribute: Final = self._settings_attribute(name) if "_resolved" in self.__dict__ else MISSING + if attribute is not MISSING: + return attribute + raise RustCoreUnsupported(f"'{name}' is not served by the Rust router core", name=name, obj=self) + + @property + def _unset_attributes(self) -> frozenset[str]: + """The instance attributes ``del`` removed, which ``_settings_attribute`` no longer serves.""" + unset: Final[object] = self.__dict__.get("_unset_attributes") + return unset if isinstance(unset, frozenset) else frozenset() # pyright: ignore[reportUnknownVariableType] # only names are stored + + def _settings_attribute(self, name: str) -> object: + """A router attribute that reads a setting, the constructor's arguments or a constant.""" + if name in self._unset_attributes: + return MISSING + r: Final = self._resolved + resolved: Final[Mapping[str, object]] = { + "num_retries": r.read("num_retries"), + "max_fallbacks": r.read("max_fallbacks"), + "timeout": r.read("timeout"), + "retry_after": r.read("retry_after"), + "explicit_allowed_fails": r.given["allowed_fails"] + if r.explicit_allowed_fails is not None and "allowed_fails" in r.given + else r.explicit_allowed_fails, + "routing_strategy_args": r.read("routing_strategy_args"), + "request_timeout": self._request_timeout, + "default_priority": self._endpoints._default_priority, # pyright: ignore[reportPrivateUsage] # parsed by the endpoint plans + } + if name in resolved: + return resolved[name] + if name in self._plain: + return self._plain[name] + if name in self._legacy: + return self._legacy[name] + if name == "healthy_deployments": + return self._hosted._core_model_list # pyright: ignore[reportPrivateUsage] # the live list, as the core keeps it + if name == "get_model_group_io_token_usage": + return self._hosted.get_model_group_io_token_usage + if name == "scheduler": + return self._endpoints._get_scheduler() # pyright: ignore[reportPrivateUsage] # the endpoint plans' scheduler + if name == "chat": + from litellm.main import Chat + + self._legacy["chat"] = Chat(params=default_litellm_params(self._config), router_obj=self) # pyright: ignore[reportArgumentType] # Chat calls the router's endpoints, which this core serves + return self._legacy["chat"] + return argument_attribute(self._config, name) + + @property + def _shaping(self) -> Shaping: + return self._endpoints.shaping(self._resolved) + + # -- building -------------------------------------------------------------------------------- + + def _new_handle(self, model_list: Sequence[object]) -> "RouterCoreHandle": + from litellm.rust_bridge._native import RouterCoreHandle + + names: Final = [ + str(name) + for entry in model_list + if isinstance(name := (as_str_mapping(entry) or {}).get("model_name"), str) + ] + default_ids: Final = self._provider_default_ids() + self._handle_default_ids = default_ids + static: Final = { + **self._resolved.static_document(model_list), + **{name: value for name, value in failover_document(self._plain).items() if value is not None}, + **({"provider_default_deployment_ids": list(default_ids)} if default_ids else {}), + } + return RouterCoreHandle(static, self._dynamic_document(names)) + + def _dynamic_document(self, model_names: Sequence[str]) -> Mapping[str, object]: + self._allowed_fails_default_sent = self._resolved.allowed_fails_is_default() + # Settings are static: an in-place edit of a live list reaches neither the walk nor its notes. + resolved: Final = self._resolved + self._walked_fallbacks = { + "fallbacks": detached(resolved.fallbacks), + "context_window_fallbacks": detached(resolved.context_window_fallbacks), + "content_policy_fallbacks": detached(resolved.content_policy_fallbacks), + } + return resolved.dynamic_document(self._version, model_names) + + def _provider_default_ids(self) -> tuple[str, ...]: + """``provider_default_deployment_ids`` as the cooldown check reads it (``deployment in + settings().provider_default_deployment_ids``): the ids it holds now. router-core holds them + with its static settings.""" + handle: Final = self._live_handle() + return () if handle is None else tuple(handle.provider_default_ids()) + + def _pick_discovery(self, *, sync: bool) -> KubernetesPodDiscovery | None: + """The pod discovery a pick resolves through: the one bound when a custom strategy was + registered (``resolve_pods_after_bound``), else the router's own (``resolve_pods_after``); + none for an async pick assigned through the deprecated attribute, which Python calls bare.""" + if not sync and callable(self.__dict__.get("_core_async_get_available_deployment")): + return None + if self._custom_strategy is not None and self._custom_discovery is not None: + return self._custom_discovery + return self._pod_discovery + + # -- the plugin context (``RoutingContext``) ----------------------------------------------- + + def _core_arm_routing_read_prefetch(self, model: str, request_kwargs: Mapping[str, object] | None = None) -> None: + """``RoutingContext.arm_read_prefetch``: declare the cooldown reads (and, for + usage-based-routing-v2, the usage reads) the pick for ``model`` makes on the request's + Redis batch, so admission's flush carries them. A miss costs nothing.""" + from litellm.router_utils.routing_read_batch import RoutingPrefetch + + try: + usage_selector: Final = self._prefetch_usage_selector(model, request_kwargs) + deployments: Final = self._hosted._core_get_model_list(model_name=model) # pyright: ignore[reportPrivateUsage] # see __init__ + if deployments: + RoutingPrefetch.arm(RoutingContext(self), usage_selector, deployments) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the prefetch reads registry() and health(); its deployments list is untyped + except Exception as e: # noqa: BLE001 # a prefetch is an optimisation, never a reason to fail the request + from litellm._logging import verbose_router_logger + + verbose_router_logger.debug( + "routing read prefetch not armed for %s: %s", + str(model).replace("\r", "").replace("\n", ""), + str(e).replace("\r", "").replace("\n", ""), + ) + + def _prefetch_usage_selector(self, model: str, request_kwargs: Mapping[str, object] | None) -> object: + """``_get_routing_context``'s selector when it is a usage-based-routing-v2 one: the request's + strategy override, else the routing group ``model`` resolves into, else the router's own.""" + from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 + + strategy, selector = self._prefetch_strategy_and_selector(model, request_kwargs or {}) + if strategy == "usage-based-routing-v2" and isinstance(selector, LowestTPMLoggingHandler_v2): + return selector + return None + + def _prefetch_strategy_and_selector(self, model: str, request: Mapping[str, object]) -> tuple[str | None, object]: + raw: Final = request.get("routing_strategy") + override: Final = normalize_strategy(raw) if isinstance(raw, str | RoutingStrategy) else None + if override is not None and override in {"simple-shuffle", *SELECTOR_ATTRIBUTES}: + return override, self._selectors.selector_for(override, "override", None, request) + groups: Final = self._hosted._core_routing_groups # pyright: ignore[reportPrivateUsage] # see __init__ + resolved: Final = self._hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # see __init__ + group_name: Final = ( + resolved + if self._hosted.get_routing_group(resolved) is not None + else next( + ( + group.group_name + for group in groups.values() + if group.routing_strategy != "priority" and resolved in group.models + ), + None, + ) + ) + if group_name is None: + return normalize_strategy(self._resolved.routing_strategy), self._selectors.default + if groups[group_name].routing_strategy == "priority": + return None, None + group_strategy: Final = normalize_strategy(groups[group_name].routing_strategy) + return group_strategy, self._selectors.selector_for(group_strategy or "", "group", group_name, None) + + async def _core_async_pre_routing_hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the request kwargs the hook stamps + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + ) -> "PreRoutingHookResponse | None": + """``RoutingContext.pre_routing_hook``: the pre-routing strategies' answer for ``model`` + (the tier group and its params), with the routing plugins run first.""" + r: Final = self._resolved + owner: Final[object] = self.__dict__.get("_core_owner") + public: Final[object] = owner() if callable(owner) else None + return await self._prerouter._hook( # pyright: ignore[reportPrivateUsage] # the core's own pre-router + model, + request_kwargs, + messages=messages, + input=input, + fallback_depth=0, + enable_tag_filtering=r.enable_tag_filtering, + match_any=r.tag_filtering_match_any, + public_router=public if public is not None else self, + run_plugins=partial(self._run_plugins, request_kwargs, messages) + if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list + else None, + ) + + def _core_log_retry( + self, + kwargs: dict[str, object], # mutable-ok: the request kwargs the record joins + e: Exception, + ) -> dict[str, object]: # mutable-ok: the same kwargs, as Python's returns them + """``RoutingContext.log_retry``.""" + return log_retry_into(kwargs, e, self._deployment_params_base_model) + + def _deployment_params_base_model(self, deployment_id: str) -> str | None: + """``_deployment_params_base_model``: the ``base_model`` a deployment declares under ``litellm_params``.""" + deployment: Final = self._hosted._core_get_deployment(model_id=deployment_id) # pyright: ignore[reportPrivateUsage] # see __init__ + if deployment is None: + return None + base_model: Final = (deployment.litellm_params.model_extra or {}).get("base_model") + return base_model if isinstance(base_model, str) else None + + def _sync_routing_state(self) -> None: + """Before a pick: sync the routing state that callers can change in place, which the Python + core reads live on every pick. This exists only for the deprecated writable attributes and + goes away with them. + + Only these cases send anything; otherwise both checks return without touching router-core:: + + router.model_names = {...} # assigning syncs at once (__setattr__) + router.model_names.add("new-group") # an in-place change is picked up here, on the next pick + router.model_name_to_deployment_indices["g"].append(3) # likewise + + router.routing_plugins.append(plugin) # first plugin: rebuilds the handle with the plugin stage + router.routing_plugins.clear() # list emptied: rebuilds without it + + Until one of the index attributes has been assigned, ``_sync_index`` sends nothing. After + that it sends the index document only when it differs from the last one it sent. + """ + self._sync_plugin_stage() + self._sync_index() + self._sync_allowed_fails() + + def _sync_allowed_fails(self) -> None: + """``_is_allowed_fails_set_on_router`` compares with the live ``litellm.allowed_fails`` on + every failure, so a write to the global can change the rule; ``_settings`` stays as copied.""" + if self._resolved.allowed_fails_is_default() != self._allowed_fails_default_sent: + names: Final = [str(d.get("model_name")) for d in self._handle.deployments()] + self._handle.apply({"settings": self._dynamic_document(names)}) + + def _sync_index(self) -> None: + """Hand router-core the registry indexes written through the deprecated attributes + (``model_names``, ``model_name_to_deployment_indices``, ``team_model_to_deployment_indices``) + as the host registry holds them now (its own deployment changes update the written objects, + as the Python core's do), whenever they differ from what it holds.""" + if not self._index_written: + return + document: Final = index_document(self._hosted, self._index_written) + if document != self._index_sent: + _ = self._handle.apply({"index": document}) + self._index_sent = document + + def _sync_plugin_stage(self) -> None: + """Give router-core the routing-plugin stage when the live plugin list (``routing_plugins``, + which callers may assign or append to) gains its first plugin, and drop it when the list + empties: the Python core checks the list on every pick.""" + live: Final = _as_list(self._hosted._core_routing_plugins) # pyright: ignore[reportPrivateUsage] # the live plugin list + if bool(live) == bool(self._resolved.routing_plugins): + return + self._resolved = dataclasses.replace(self._resolved, routing_plugins=tuple(live)) # pyright: ignore[reportArgumentType] # the plugins as listed + self._rebuild() + + def _rebuild(self) -> None: + """A new Rust core for changed static settings, over the same deployments and the + providers the old one's builds resolved for them.""" + old: Final = self._handle + model_list: Final = list(self._hosted._core_model_list) # pyright: ignore[reportPrivateUsage] # see __init__ + self._handle = self._new_handle(model_list) + self._handle.adopt_providers(old) + old.retire() + # The live list and indexes read the new handle's (the same entries, kept as the same objects). + self._hosted.refresh() + # The new handle builds its indexes from the deployments: written ones are handed again. + self._index_sent = None + + def _run_build( + self, + entries: Sequence[object], + path: str, + resolve_secrets: bool, + install: BuildInstall, + answer: Callable[[Mapping[str, object]], object], + ) -> None: + """Drive one router-core deployment build (``RouterCoreHandle.build``), answering each of + its steps with ``answer``; router-core installs each deployment as it accepts it, and the + host registry reads the list and indexes back before answering the next step.""" + call: Final = self._handle.build(_plain(list(entries)), path, resolve_secrets, install) # pyright: ignore[reportArgumentType] # plain JSON entries + outcome: object = None # rebind-ok: each step's answer + while True: + step = call.next(outcome) + self._hosted.refresh() + if step["kind"] == "done": + break + if step["kind"] == "failed": + raise ValueError(str(step.get("error"))) + outcome = _plain(answer(step)) + + def _live_handle(self) -> "RouterCoreHandle | None": + """The handle the live model_list reads, once the constructor made one.""" + from litellm.rust_bridge._native import RouterCoreHandle + + handle: Final[object] = self.__dict__.get("_handle") + return handle if isinstance(handle, RouterCoreHandle) else None + + def _apply_registry_changes(self, changes: Mapping[str, object]) -> None: + """Apply a host registry write router-core's build did not make (a removal is a delete, a + swapped list a replace) as a changeset.""" + _ = self._handle.apply(_plain(dict(changes))) # pyright: ignore[reportArgumentType] # a plain JSON changeset + + def _sync_registry(self) -> None: + """After a host registry write: router-core already holds it (the build installed what it + added, a removal came as a changeset), so only the settings it feeds move.""" + from litellm.rust_bridge._native import RouterCoreHandle + + self._tag_filtering_models = None + handle: Final[object] = self.__dict__.get("_handle") + # A wildcard deployment added an id the cooldown check skips. + if isinstance(handle, RouterCoreHandle) and self._provider_default_ids() != self._handle_default_ids: + self._rebuild() + + # -- entry points ---------------------------------------------------------------------------- + + def registry(self) -> RouterRegistry: + """The registry view, built once (``registry()`` returns the same object each time).""" + if self._registry_view is None: + self._registry_view = hosted_registry_view(self._hosted) + return self._registry_view + + def settings(self) -> RustSettings: + """A cached snapshot, replaced when a changeset changes a setting.""" + if self._settings is None: + self._settings = self._settings_snapshot() + return self._settings + + def _settings_snapshot(self) -> RustSettings: + r: Final = self._resolved + snapshot: Final = RustSettings( + routing_strategy=r.routing_strategy, + fallbacks=r.read("fallbacks"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + context_window_fallbacks=r.read("context_window_fallbacks"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + content_policy_fallbacks=r.read("content_policy_fallbacks"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + default_deployment=r.default_deployment, + default_litellm_params=dict(r.default_litellm_params) + if "default_litellm_params" in r.assigned + else default_litellm_params(self._config), + default_max_parallel_requests=self._endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given + cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + allowed_fails=r.read("allowed_fails"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + allowed_fails_policy=r.allowed_fails_policy, # pyright: ignore[reportArgumentType] # the policy as passed + disable_cooldowns=r.read("disable_cooldowns"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + enable_pre_call_checks=r.enable_pre_call_checks, + enable_health_check_routing=r.health_routing.enable_health_check_routing, + enable_tag_filtering=r.read("enable_tag_filtering"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + tag_filtering_match_any=r.read("tag_filtering_match_any"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + tag_routing_prefix=r.read("tag_routing_prefix"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + fallback_access_check=r.fallback_access_check, # pyright: ignore[reportArgumentType] # the check as passed + fallback_budget_check=r.fallback_budget_check, # pyright: ignore[reportArgumentType] # the check as passed + cache_responses=self._endpoints.cache_responses, + search_tools=self._endpoints.search_tools, # pyright: ignore[reportArgumentType] # the configured entries, as passed + guardrail_list=self._endpoints.guardrail_list, # pyright: ignore[reportArgumentType] # same + provider_default_deployment_ids=self._hosted._core_provider_default_deployment_ids, # pyright: ignore[reportPrivateUsage] # the host registry's live list + health_check_ignore_transient_errors=r.health_routing.health_check_ignore_transient_errors, + background_health_check_model_groups=r.health_routing.background_health_check_model_groups, + router_general_settings=self._general_settings, # pyright: ignore[reportArgumentType] # the setting as given + provider_budget_config=self._checks.provider_budget_config, # pyright: ignore[reportArgumentType] # the budget limiter's live config + _router=self, # pyright: ignore[reportArgumentType] # the snapshot reads as_dict from this core + kubernetes_pod_discovery=self._pod_discovery, + ) + if not self._deleted: + return snapshot + # A deleted setting reads as unset. + return dataclasses.replace( + snapshot, + **dict.fromkeys(self._deleted.intersection(SETTING_FIELDS), None), # pyright: ignore[reportArgumentType] # settings fields only: an alias map or retry count is deletable too + _unset=snapshot._unset | self._deleted, # pyright: ignore[reportPrivateUsage] # the snapshot's own unset set + ) + + def available_guardrail(self, name: str) -> object: + return self._endpoints.available_guardrail(name) + + def settings_dict(self) -> Mapping[str, object]: + r: Final = self._resolved + latency: Final = self._selectors.latency_args() if r.routing_strategy == "latency-based-routing" else None + # `get_settings` reports the latency selector's full routing args. + args: Final = ( + latency if latency is not None else r.given.get("routing_strategy_args", dict(r.routing_strategy_args)) + ) + values: Final = { + "routing_strategy_args": args, + "routing_strategy": r.routing_strategy, + "allowed_fails": r.read("allowed_fails"), + "cooldown_time": self._cooldown_given, + "num_retries": r.read("num_retries"), + "timeout": r.read("timeout"), + "retry_after": r.read("retry_after"), + "fallbacks": r.read("fallbacks"), + "context_window_fallbacks": r.read("context_window_fallbacks"), + "model_group_alias": dict(r.model_group_alias), + "enable_mid_stream_fallback_continuation": r.enable_mid_stream_fallback_continuation, + "enable_tag_filtering": r.read("enable_tag_filtering"), + "tag_routing_prefix": r.read("tag_routing_prefix"), + "routing_groups": group_settings(r.routing_groups), + } + return settings_view(values, self._plain, self._deleted) + + def health(self) -> RustHealth: + """The health view, built once.""" + if self._health_view is None: + self._health_view = RustHealth(self) + return self._health_view + + def observe(self, logging_obj: "LiteLLMLogging | None" = None, *, request_scoped: bool = False) -> RoutingObservers: + """The routing observers: the Slack alerting logger ``alerting_config`` built, if any. + ``request_scoped=True`` takes the router's callbacks off the host lists; passing a request's + ``logging_obj`` binds its selectors and policies to it. This core registers no outcome + callbacks, so it binds none either: router-core reads what they would do for the request + (``OptionalChecks.outcome_callbacks``).""" + if request_scoped: + self._serve_through_request_logging() + if logging_obj is not None: + self._checks.bind_to_request_logging(logging_obj) + return RoutingObservers(slack_alerting_logger=self._slack_alerting_logger) + + def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: + """``_core_set_custom_routing_strategy``: ``plugin`` picks every deployment from now on, until + the routing strategy changes.""" + self._custom_strategy = plugin + # `resolve_pods_after_bound`: its picks resolve pods through the discovery bound now. + self._custom_discovery = self._pod_discovery + # Python assigns the strategy's picks over its own: a caller of the pick reaches the + # strategy with the arguments it passed and no others. + self.__dict__["get_available_deployment"] = self._custom_sync_pick(plugin) + # It replaces an async pick assigned through the deprecated attribute, as Python's does. + if "_core_async_get_available_deployment" in self.__dict__: + del self.__dict__["_core_async_get_available_deployment"] + + def _custom_sync_pick(self, plugin: "CustomRoutingStrategyBase") -> object: + discovery: Final = self._custom_discovery + pick: Final = plugin.get_available_deployment + return pick if discovery is None else resolve_pods_after_bound(pick, discovery) + + def _custom_async_pick(self) -> object: + """``_core_set_custom_routing_strategy``'s ``_core_async_get_available_deployment``.""" + custom: Final = self._custom_strategy + discovery: Final = self._custom_discovery + pick: Final = custom.async_get_available_deployment # pyright: ignore[reportOptionalMemberAccess] # __getattr__ checked it + return pick if discovery is None else async_resolve_pods_after_bound(pick, discovery) + + def set_custom_routing_strategy(self, CustomRoutingStrategy: "CustomRoutingStrategyBase") -> None: + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the deprecated alias notes its use like Python's + ) + + _note("set_custom_routing_strategy", "register_plugin(strategy)") + self.register_plugin(CustomRoutingStrategy) + + def retire(self) -> None: + """``_core_discard``: only the router's global registrations go; it keeps serving calls.""" + untrack_live_registry(self._hosted) + self._retire_callbacks() + if self._hosted.strategies is not None: + self._hosted.strategies.retire() + + def apply(self, changes: ChangeSet) -> ApplyResult: + """``apply_changes``: assigned settings, then ``update_settings``, upserts, deletes and the + Redis cache, in that order. An unknown name refuses the changeset; a setting + ``update_settings`` raises for ends it there, with the settings before it applied.""" + from litellm.router_api import ASSIGNABLE_ALIASES, PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS + + if changes.refresh_model_info: + raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()") + unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES}) + if unknown: + raise ValueError(f"not assignable router settings: {unknown}") + unserved: Final = sorted(set(changes.assign) - _STATIC_SETTINGS - _DYNAMIC_SETTINGS - _HOST_SETTINGS) + if unserved: + raise RustCoreUnsupported(f"the Rust router core does not serve the setting(s) {', '.join(unserved)} yet") + update: Final = read_settings_update(changes.settings, self._resolved.routing_strategy) + # `apply_changes` assigns before it runs `update_settings`: a budget config assigned beside + # `optional_pre_call_checks` builds the budget limiter with the new config. + self._assign_host_settings(changes.assign) + if update.optional_pre_call_checks is not MISSING: + self._checks.set(update.optional_pre_call_checks) + if update.continuation is not MISSING: + self._set_continuation(continuation_flag(update.continuation)) + self._apply_settings(changes.assign, update) + if update.failure is not None: + raise update.failure + # Upserts first, then deletes (`apply_changes`), through the host registry, which hands + # each change to the Rust core. + upserted: Final = [self._hosted._core_upsert_deployment(deployment=d) for d in changes.upsert] # pyright: ignore[reportPrivateUsage] # see __init__ + deleted: Final = [self._hosted._core_delete_deployment(id=i) for i in changes.delete] # pyright: ignore[reportPrivateUsage] # see __init__ + if changes.redis_cache is not None: + # `_core_update_redis_cache`: the routing state's cache gains the Redis cache. + self._cache.attach_redis_cache(changes.redis_cache) + self._checks.prompt_caching_pin_cache.attach_redis_cache(changes.redis_cache) + return ApplyResult( + version=self._hosted._registry_version, # pyright: ignore[reportPrivateUsage] # see __init__ + upserted=upserted, + deleted=deleted, # pyright: ignore[reportArgumentType] # the removed model_list entries, as the core returns them + ) + + async def aapply(self, changes: ChangeSet) -> ApplyResult: + """``apply``, then the model info refresh, which needs I/O.""" + result: Final = self.apply(dataclasses.replace(changes, refresh_model_info=False, refresh_client=None)) + if changes.refresh_model_info: + await self._hosted.arefresh_model_info(changes.refresh_client) + return result + + def _apply_settings(self, assign: Mapping[str, object], update: SettingsUpdate) -> None: + values: Final = { + **without(assign, *_HOST_SETTINGS), + **update.values, + } + given: Final = dict(update.values) # the settings passed through `update_settings` + groups_input: Final[object] = ( + update.routing_groups if update.routing_groups is not MISSING else self._resolved.routing_groups_input + ) + # What `_core_update_settings` runs after its loop, which a setting that raised never reaches. + settled: Final = update.failure is None + groups_failure: Exception | None = None # rebind-ok: set when the groups passed do not parse + if settled and (update.routing_groups is not MISSING or self._changes_strategy(update.values)): + # `update_settings` re-inits the routing groups from the groups passed, else (on a + # strategy change) from the groups last passed, whatever a raw write assigned since. + try: + values["routing_groups"] = parse_groups( + groups_input, + frozenset(str(d.get("model_name")) for d in self._handle.deployments()), + frozenset(self._resolved.model_group_alias), + ) + except Exception as raised: # noqa: BLE001 # raised below: `_init_routing_groups` runs after the loop + groups_failure = raised # rebind-ok: None until the groups fail + else: + given["routing_groups"] = groups_input + self._store_plain(update.plain) + if self._deleted & set(values): + self._deleted = self._deleted - set(values) + self._settings = None + if values: + previous: Final = self._resolved + resolved: Final = dataclasses.replace( + previous.changed(values, frozenset(given)), + routing_groups_input=groups_input if "routing_groups" in values else previous.routing_groups_input, + ) + static_changed: Final = bool(_STATIC_SETTINGS & set(values)) + self._resolved = resolved + if set(values) & set(SETTING_FIELDS): + self._settings = None # the snapshot holds settings fields, not the plain attributes + if "model_group_alias" in values: + self._hosted._core_model_group_alias = dict(resolved.model_group_alias) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # see __init__ + self._apply_strategy(previous, resolved, given, settled=settled) + if "routing_groups" in values: + self._settings = None + self._hosted.set_routing_groups(resolved.routing_groups) + if "cooldown_time" in values: + self._cooldown_given = values["cooldown_time"] + if values["cooldown_time"] is None: + # `add_deployment_to_cooldown(cooldown_time=None)` cools down for the cooldown + # cache's default, which keeps the constructor's value. + self._resolved = dataclasses.replace( + resolved, cooldown_time=float(self._cooldowns.default_cooldown_time) + ) + if static_changed: + self._rebuild() + else: + names: Final = [str(d.get("model_name")) for d in self._handle.deployments()] + self._handle.apply({"settings": self._dynamic_document(names)}) + if groups_failure is not None: + raise groups_failure + + def _store_plain(self, plain: Mapping[str, object]) -> None: + """Settings the Python core keeps as plain attributes, read from the next call on.""" + flags_before: Final = failover_document(self._plain) + self._plain.update(plain) + if failover_document(self._plain) != flags_before: + self._rebuild() + policies: Final = {name: plain[name] for name in _RETRY_POLICY_SETTINGS if name in plain} + if policies: + self._set_retry_policies(policies) + + def _changes_strategy(self, updated: Mapping[str, object]) -> bool: + """``update_settings`` names a ``routing_strategy`` other than the router's.""" + if "routing_strategy" not in updated: + return False + return _strategy_name(updated["routing_strategy"]) != self._resolved.routing_strategy + + def _assign_host_settings(self, assign: Mapping[str, object]) -> None: + """The assigned settings only the host reads, each stored as given (detached, as + ``apply_changes`` stores it) where its readers find it from the next call on.""" + given: Final = {name: detached(value) for name, value in assign.items() if name in _HOST_SETTINGS} + if not given: + return + for name, value in given.items(): + if name in ("enable_health_check_routing", "background_health_check_model_groups"): + self._resolved = dataclasses.replace( + self._resolved, + health_routing=dataclasses.replace(self._resolved.health_routing, **{name: value}), # pyright: ignore[reportArgumentType] # stored as given, as the Python core stores it + ) + elif name == "provider_default_deployment_ids": + # The list wildcard deployments added later are appended to, as on the Python core. + self._hosted._core_provider_default_deployment_ids = value # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # stored as given, as the Python core stores it + elif name == "cache_responses": + self._endpoints.set_cache_responses(bool(value)) + elif name == "provider_budget_config": + # Read when the budget limiter is built; a limiter built already keeps its config. + self._checks.provider_budget_config = value + elif name == "health_check_ignore_transient_errors": + self._resolved = dataclasses.replace( + self._resolved, + health_routing=dataclasses.replace( + self._resolved.health_routing, + health_check_ignore_transient_errors=value, # pyright: ignore[reportArgumentType] # stored as given, as the Python core stores it + ), + ) + elif name == "router_general_settings": + self._general_settings = value + elif name == "kubernetes_pod_discovery": + self._pod_discovery = value # pyright: ignore[reportAttributeAccessIssue] # stored as given, as the Python core stores it + else: + self._endpoints.assign_setting(name, value) + self._deleted = self._deleted - set(given) + self._settings = None + if {"enable_health_check_routing", "background_health_check_model_groups"} & set(given) or ( + self._provider_default_ids() != self._handle_default_ids + ): + self._rebuild() + + def _set_retry_policies(self, policies: Mapping[str, object]) -> None: + """The ``retry_policy`` and ``model_group_retry_policy`` the retry loop reads from then on, + from ``update_settings`` or an attribute write: router-core holds them with its static + settings.""" + previous: Final = self._resolved + resolved: Final = dataclasses.replace( + previous, + retry_policy=readable_retry_policy(policies.get("retry_policy", previous.retry_policy)), + model_group_retry_policy=policies.get("model_group_retry_policy", previous.model_group_retry_policy), + ) + if resolved != previous: + self._resolved = resolved + self._rebuild() + + def _apply_strategy( + self, previous: Resolved, resolved: Resolved, values: Mapping[str, object], *, settled: bool + ) -> None: + """``_core_update_settings``: a changed strategy re-inits the selectors with the args passed + alongside it (or none); changed args alone rebuild the default selector. What runs after + the loop needs ``settled``: no setting raised, and the groups are in ``values``.""" + if "routing_strategy" in values and resolved.routing_strategy != previous.routing_strategy: + if resolved.routing_strategy == "lar1": + # `_switch_routing_strategy`: LAR-1 is a custom strategy, registered in place of + # `routing_strategy_init`, so the selectors the router held stay as they were. + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy( + RoutingContext(self), # pyright: ignore[reportArgumentType] # the core's RoutingContext + as_str_mapping(values.get("routing_strategy_args")), + ) + else: + # `routing_strategy_init` drops a custom strategy (`_reset_custom_routing_strategy`), + # and an async pick assigned through the deprecated attribute. + self._custom_strategy = None + self.__dict__.pop("get_available_deployment", None) + if "_core_async_get_available_deployment" in self.__dict__: + del self.__dict__["_core_async_get_available_deployment"] + self._selectors.init_default( + resolved.routing_strategy, as_str_mapping(values.get("routing_strategy_args")) or {} + ) + if "routing_groups" in values: + self._selectors.init_groups(_group_selectors(resolved.routing_groups)) + elif "routing_groups" in values: + self._selectors.init_groups(_group_selectors(resolved.routing_groups)) + elif ( + settled + and "routing_strategy_args" in values + and resolved.routing_strategy_args != previous.routing_strategy_args + ): + if resolved.routing_strategy == "lar1": + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy(RoutingContext(self), resolved.routing_strategy_args) # pyright: ignore[reportArgumentType] # the core's RoutingContext + return + self._selectors.rebuild_default(resolved.routing_strategy_args) + + def upsert_deployment(self, deployment: Deployment) -> Deployment | None: + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the router's deprecation notice + ) + + _note("upsert_deployment", "apply(ChangeSet(upsert=[deployment]))") + return self.apply(ChangeSet(upsert=[deployment])).upserted[0] + + def delete_deployment(self, id: str) -> Deployment | None: + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see above + + _note("delete_deployment", "apply(ChangeSet(delete=[id]))") + return self.apply(ChangeSet(delete=[id])).deleted[0] + + def update_settings(self, **kwargs: object) -> None: # kwargs-ok: the deprecated signature, unchanged + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("update_settings", "apply(ChangeSet(settings={...}))") + _ = self.apply(ChangeSet(settings=kwargs)) + + def update_redis_cache(self, cache: "RedisCache") -> None: + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))") + _ = self.apply(ChangeSet(redis_cache=cache)) + + def get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("get_model_list", "registry().deployments(model_name, team_id=...)") + return self._hosted._core_get_model_list(model_name=model_name, team_id=team_id) # pyright: ignore[reportPrivateUsage, reportReturnType] # the host registry's listing, as the deprecated member returns it + + @staticmethod + def get_valid_args() -> list[str]: # mutable-ok: the old return type, a fresh list + """The arguments ``Router.__init__`` accepts.""" + return list(ROUTER_INIT_ARGS) + + async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None: + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))") + _ = await self.aapply(ChangeSet(refresh_model_info=True, refresh_client=client)) + + def flush_cache(self) -> None: + """Drop the host's response cache and the routing state.""" + self._host.set_global_cache(None) + self._cache.flush_cache() + self._checks.prompt_caching_pin_cache.in_memory_cache.flush_cache() + + def reset(self) -> None: + """Empty the host's success and failure callback lists (every entry, not only this core's), + drop the retry policy and flush the caches.""" + for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"): + self._host.set_callback_list(name, []) + self._plain["retry_policy"] = None + self._set_retry_policies({"retry_policy": None}) + self.flush_cache() + + def __delattr__(self, name: str) -> None: + # `del router.` unsets it: the settings view reports it unset from then on. + from litellm.router_api import SETTING_FIELDS + + # The deprecated attribute deletes the core's own name (``_core_``). + setting: Final = name.removeprefix("_core_") + if setting not in SETTING_FIELDS and setting not in _DELETABLE_ATTRIBUTES: + if name in self.__dict__ or name in self._unset_attributes: + object.__delattr__(self, name) + return + if self._settings_attribute(name) is MISSING: + object.__delattr__(self, name) + return + # An attribute the Python core keeps on the instance (`alerting_config`, an argument + # as passed): reading it raises AttributeError from now on. + self.__dict__["_unset_attributes"] = self._unset_attributes | {name} + return + if setting in self._deleted: + raise self._no_attribute(setting) + self._deleted = self._deleted | {setting} + self._settings = None + if setting == "model_group_alias": + del self._hosted._core_model_group_alias # pyright: ignore[reportPrivateUsage] # every read of it raises from now on, as on the Python core + elif setting == "scheduler": + self._endpoints.delete_scheduler() + + def _no_attribute(self, name: str) -> AttributeError: + """What reading an attribute deleted from the core raises (the facade renames it).""" + return AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'", name=name, obj=self) + + def _no_router_attribute(self, name: str) -> AttributeError: + """``_no_attribute`` for a read through ``registry()``, which the deprecated attribute + renames when it is the router's own.""" + owner: Final[object] = self.__dict__.get("_core_owner") + public: Final[object] = owner() if callable(owner) else None + return AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'", + name=name, + obj=self if public is None else public, + ) + + def _refuse_deleted_reads(self, request: Mapping[str, object]) -> None: + """What a call reads of the router before any attempt, once deleted: routing reads + ``_core_model_group_alias``, and ``async_function_with_retries`` reads ``self.num_retries`` + when the request names none. Every hop raises it, so no fallback serves the call.""" + if "model_group_alias" in self._deleted: + raise self._no_attribute("_core_model_group_alias") + if "num_retries" in self._deleted and request.get("num_retries") is None: + raise self._no_attribute("num_retries") + + def __setattr__(self, name: str, value: object) -> None: + # A setting write is a changeset (`router.num_retries = 2` is `apply(ChangeSet(assign=...))`), + # a `model_list` write swaps the live list, and any other attribute is kept as written. + from litellm.router_api import ASSIGNABLE_ALIASES, PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS + + if name in PLAIN_SETTING_ATTRIBUTES or name in SETTING_FIELDS or name in ASSIGNABLE_ALIASES: + _ = self.apply(ChangeSet(assign={name: value})) + elif name in PLAIN_DEFAULTS or name == "max_retries": + # Plain attributes on the Python core, outside the setting lists: stored as given. + self._store_plain({name: value}) + elif name == "scheduler": + self._endpoints.set_scheduler(value) + self._deleted = self._deleted - {name} + elif name in ("model_list", "_core_model_list"): + # `router.model_list = [...]` reaches the core as `_core_model_list` through the shim. + self._assign_model_list(_as_list(value)) + elif name in _HOSTED_STATE: + # A deprecated attribute write of the registry state: it lands where this core keeps it. + setattr(self._hosted, name, value) + if name in INDEX_ATTRIBUTES: + self._index_written = self._index_written | {name} + self._sync_index() + elif name == "_core_routing_groups": + # The Python core routes on the assigned groups from the next call on, while its + # member map and group selectors stay as they were; router-core holds the groups and + # the member map with its static settings. + assigned: Final = as_str_mapping(value) or {} + previous: Final = self._resolved + self._resolved = dataclasses.replace( + previous, + routing_groups=tuple(g for g in assigned.values() if isinstance(g, RoutingGroup)), + routing_group_members=previous.routing_group_members + if previous.routing_group_members is not None + else member_map(previous.routing_groups), + ) + self._settings = None + self._rebuild() + elif name == "_core_routing_plugins": + self._sync_plugin_stage() + elif name == "_core_prompt_caching_pin_cache": + # Read when a prompt caching check is built, by `update_redis_cache` and by `flush_cache`. + self._checks.prompt_caching_pin_cache = value # pyright: ignore[reportAttributeAccessIssue] # stored as given, as the Python core stores it + elif name in _REBOUND_STATE: + # A deprecated attribute write of the routing state (``router.cache = ...``): this core + # reads the new object from now on, as the Python core reads its rebound attribute. + object.__setattr__(self, _REBOUND_STATE[name], value) + if name == "_core_cache": + self._hosted._core_cache = value # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # usage reads come from the routing cache + elif name == "_core_slack_alerting_logger": + from litellm.integrations.custom_logger import CustomLogger + + # `_slack_alerting_loggers` reads the attribute each time: the new logger is the one bound. + self._checks.slack_alerting_loggers = (value,) if isinstance(value, CustomLogger) else () + else: + # Any other attribute is kept as written; `_core_async_get_available_deployment` (the + # deprecated async pick) is then asked by every async pick (see `_pre_route`). + object.__setattr__(self, name, value) + + async def select( # pyright: ignore[reportIncompatibleMethodOverride] # the RouterCore selection + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the RouterCore signature + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + input: object = None, + health_check_probe: bool = False, + **_: object, # kwargs-ok: the RouterCore select options this core ignores + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + """Pick a deployment for ``model`` without calling it (``pick``), list the deployments a + pick would choose among (``healthy``), or pick one for a pass-through route + (``pass_through``, which picks with the sync strategies, as Python does).""" + if kind not in ("pick", "healthy", "pass_through"): + raise ValueError(f"unknown select kind {kind!r}") + self._refuse_deleted_reads({"num_retries": 0}) # a pick reads no retry budget + request: Final = request_kwargs if request_kwargs is not None else {} + selection: Final = _Selection( + self, + model, + request, + kind, + specific_deployment, + messages, + input, + probe=kind == "healthy" and health_check_probe, + ) + # A pick that makes no call holds no least-busy reservation (`select` on the Python core). + with picks_reserve(False): + if kind == "pass_through": + return selection.run_sync() + return await selection.run() + + async def async_get_available_deployment_for_pass_through( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the legacy signature + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + ) -> Mapping[str, object]: + """The full healthy chain, then only ``use_in_pass_through`` deployments, picked with the + async strategies.""" + selection: Final = _Selection( + self, model, request_kwargs, "async_pass_through", specific_deployment, messages, input + ) + picked: Final = await selection.run() + return picked if isinstance(picked, Mapping) else {} + + def get_available_deployment( + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the legacy signature + ) -> Mapping[str, object]: + """The sync pick: no pre-routing, the sync strategies.""" + selection: Final = _Selection( + self, + model, + request_kwargs if request_kwargs is not None else {}, + "sync_pick", + specific_deployment, + messages, + input, + ) + picked: Final = selection.run_sync() + return picked if isinstance(picked, Mapping) else {} + + def get_available_deployment_for_pass_through( + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the legacy signature + ) -> Mapping[str, object]: + """Deprecated: ``select(model, kind="pass_through")``.""" + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the router's deprecation notice + ) + + _note("get_available_deployment_for_pass_through", 'select(model, kind="pass_through")') + selection: Final = _Selection( + self, + model, + request_kwargs if request_kwargs is not None else {}, + "pass_through", + specific_deployment, + messages, + input, + ) + picked: Final = selection.run_sync() + return picked if isinstance(picked, Mapping) else {} + + # -- endpoints ------------------------------------------------------------------------------- + + async def call(self, op: str, request: Mapping[str, object]) -> object: + """Run one endpoint with the request's keyword payload, as ``getattr(core, op)(**request)``.""" + member: Final[object] = getattr(self, op) # pyright: ignore[reportAny] # litellm's untyped helpers and request values + if not callable(member): + raise TypeError(f"{op!r} is not an endpoint") + result: Final[object] = member(**request) + return await result if inspect.isawaitable(result) else result + + async def acompletion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.acompletion(request_from_call("acompletion", args, kwargs)) + + async def atext_completion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.atext_completion(request_from_call("atext_completion", args, kwargs)) + + async def aembedding(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._drive(plan_for("aembedding"), request_from_call("aembedding", args, kwargs)) + + async def aresponses(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.generic(plan_for("aresponses"), request_from_call("aresponses", args, kwargs)) + + async def aanthropic_messages(self, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.generic(plan_for("aanthropic_messages"), kwargs) + + async def anthropic_messages(self, **kwargs: object) -> object: # kwargs-ok: endpoint payload + """The proxy's ``/v1/messages`` route name for ``aanthropic_messages``; async, as in PythonCore.""" + return await self._endpoints.generic(plan_for("anthropic_messages"), kwargs) + + async def aimage_generation(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._drive(plan_for("aimage_generation"), request_from_call("aimage_generation", args, kwargs)) + + async def atranscription(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._drive(plan_for("atranscription"), request_from_call("atranscription", args, kwargs)) + + async def aspeech(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + request: Final = dict(request_from_call("aspeech", args, kwargs)) + request.setdefault("voice", None) + return await self._drive(plan_for("aspeech"), request) + + async def arerank(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + # `arerank` sets kwargs["input"] = input: the Python builtin, which reaches the deployment call. + request: Final = {**request_from_call("arerank", args, kwargs), "input": input} + return await self._drive(plan_for("arerank"), request) + + async def amoderation(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.amoderation(request_from_call("amoderation", args, kwargs)) + + def completion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return self._call_sync(plan_for("completion"), request_from_call("completion", args, kwargs)) + + def text_completion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return self._endpoints.text_completion(request_from_call("text_completion", args, kwargs)) + + def embedding( + self, + model: str, + input: str | list[object], # mutable-ok: the request field, as the Python core takes it + is_async: bool | None = False, + **kwargs: object, # kwargs-ok: endpoint payload + ) -> object: + """``embedding``'s own signature; as there, ``is_async`` is named and goes no further.""" + return self._call_sync(plan_for("embedding"), {**kwargs, "model": model, "input": input}) + + def image_generation(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return self._call_sync(plan_for("image_generation"), request_from_call("image_generation", args, kwargs)) + + # -- driving a call -------------------------------------------------------------------------- + + def _start( + self, + plan: Plan, + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + prepared_already: bool = False, + ) -> "tuple[RouterCall[_CallSteps], Mapping[str, object]]": + self._refuse_deleted_reads({**request, **(overrides or {})}) + self._sync_routing_state() + model: Final = request.get("model") + if model is None and "model" in request: + raise self._no_model_error(plan, request) + if "model" not in request: + raise TypeError(f"{plan.endpoint}() needs a model name") + name: Final = model if isinstance(model, str) else self.routed_name(model) + field: Final = plan.prepare_field or "metadata" + if request.get(field) is not None and not isinstance(request.get(field), Mapping): + raise TypeError(f"{field} must be a mapping") + prepared: Final = dict(request) if prepared_already else self._endpoints.prepare(plan, request, model) # pyright: ignore[reportArgumentType] # the name as given, which the metadata keeps + return self._handle.start( + plan.endpoint, self._routing_inputs(plan, request, prepared, name, overrides) + ), prepared + + def service_logged(self, call_type: str, start_time: float, end_time: float, request: Mapping[str, object]) -> None: + """The ``ServiceTypes.ROUTER`` success event of a timed step, as a task of the running loop.""" + import asyncio + + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + from litellm.types.services import ServiceTypes + + _ = asyncio.create_task( # fire and forget, as the Python core schedules it + self.service_logger_obj.async_service_success_hook( # pyright: ignore[reportUnknownMemberType] # its unused `event_metadata` is an untyped dict + service=ServiceTypes.ROUTER, + duration=end_time - start_time, + call_type=call_type, + start_time=start_time, + end_time=end_time, + parent_otel_span=get_parent_otel_span_from_kwargs(request), + ) + ) + + def track_deployment_pick(self, deployment_id: object, request: Mapping[str, object]) -> None: + """``_track_deployment_metrics`` for a pick: ``_update_usage``'s count of the deployment's + picks in the router cache, on this worker only. A default deployment served under a + placeholder id counts under the id Python's copy of it holds.""" + from litellm._logging import verbose_router_logger + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + + parent_otel_span: Final = get_parent_otel_span_from_kwargs(request) + try: + model_id: Final = ( + self._default_deployment_id() + if deployment_id in (DEFAULT_DEPLOYMENT_FAILING_ID, DEFAULT_DEPLOYMENT_COPIED_ID) + else deployment_id + ) + if model_id is None: + return + counts: Final = self._pick_counts() + count: Final = counts.get_cache(key=model_id, parent_otel_span=parent_otel_span, local_only=True) + if count is None: + counts.set_cache(key=model_id, value=1, local_only=True, ttl=60) # only store for 60s + else: + counts.set_cache(key=model_id, value=count + 1, local_only=True) + except Exception as e: # noqa: BLE001 # as `_track_deployment_metrics` catches it + verbose_router_logger.error("Error in _track_deployment_metrics: %s", e) + + def _pick_counts(self) -> _PickCounts: + return self._cache + + def _default_deployment_id(self) -> object: + return self._resolved.default_deployment.get("model_info", {}).get("id", None) # pyright: ignore[reportOptionalMemberAccess, reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # read as Python reads it + + def routed_name(self, model: object, *, read_as_member: bool = False) -> str: + """The name a ``model`` that is no string routes under, for the native calls, which take strings + alone. Python's first reads raise for most values: the alias lookup hashes it (a sync pick reads + ``model in _core_model_names`` first, ``read_as_member``), then ``_core_get_model_list`` reaches + ``declared_provider``, which searches it. A value both accept (falsy, a tuple) is a name no + deployment serves, and ``str(model)`` is how every error names it.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + if read_as_member: + _ = model in frozenset[object]() + else: + _ = self._hosted._core_get_model_from_alias(model) # pyright: ignore[reportPrivateUsage, reportArgumentType] # raising is the point + _ = inferred_provider(model) # pyright: ignore[reportArgumentType] # see above + return str(model) + + def _no_model_error(self, plan: Plan, request: Mapping[str, object]) -> BaseException: + """``model=None``: the pick fails in ``_common_checks_available_deployment``, naming the + first default fallback when there is one, and with no group the fallback wrapper raises + that error as it is (``async_function_with_fallbacks_common_utils``).""" + import litellm.exceptions as litellm_exceptions + + named: Final = next( + ( + targets[0] + for entry in _as_list(self._resolved.fallbacks) + if (targets := _as_list((as_str_mapping(entry) or {}).get("*"))) + ), + None, + ) + error: Final = litellm_exceptions.BadRequestError( + message=f"You passed in model={named}. {RouterErrors.no_healthy_deployments.value}", + model=named, # pyright: ignore[reportArgumentType] # None, as the Python router passes it + llm_provider="", + ) + if not plan.sync: + _Driver(self, plan, request)._log_pick_failure(error) # pyright: ignore[reportPrivateUsage] # the async pick's own logging + return error + + def _routing_inputs( + self, + plan: Plan, + request: Mapping[str, object], + prepared: Mapping[str, object], + model: str, + overrides: Mapping[str, object] | None, + ) -> Mapping[str, object]: + """The document router-core starts a call from: what it routes on, then what this call changes.""" + fields: dict[str, object] = {"model": model} # mutable-ok: built up below, later keys win # rebind-ok: same + fields.update(self._routing_fields(prepared, plan.pick_input(prepared)[0])) + fields.update(pinned_video_deployment(plan, request)) + fields.update(unrouted_fields(plan)) + fields.update(failover_fields(plan)) + fields.update(scoped_fallback_fields(plan, request)) + fields.update(self._selection_fields(plan, prepared)) + fields.update(self._outcome_fields(plan, request)) + # The sync path never runs pre-routing; a custom strategy picks on both. + if not plan.sync or self._custom_strategy is not None: + fields["pre_routes"] = True + if (mock := mock_failure(request)) is not None: + fields["mock_failure"] = mock + fields.update(self._retry_policy_raises(request)) + if overrides is not None and "num_retries" in overrides: + fields.pop("uncounted_retries", None) + fields.update(overrides or {}) + return _document(fields) + + def _outcome_fields(self, plan: Plan | None, request: Mapping[str, object]) -> Mapping[str, object]: + """Which outcome callbacks run for the request: an endpoint that makes one attempt skips + ``make_call``, and a recovered stream is async.""" + sync: Final = plan is not None and plan.sync + make_call: Final = plan is None or not plan.single_attempt + callbacks: Final = self.outcome_callbacks(request, sync=sync, make_call=make_call) + return {"outcome_callbacks": callbacks} if callbacks else {} + + def _selection_fields(self, plan: Plan, request: Mapping[str, object]) -> Mapping[str, object]: + """What an endpoint's pick reads beyond the routing fields: the narrowing keys the caller + passed, the endpoint ``endpoint_selection`` names, and whether it asks for web search, + which only an async pick filters by.""" + endpoint: Final = selection_endpoint(self._port, plan) + web_search: Final = not plan.sync and self._picks_async() and is_web_search_request(request) + # The generic handler `_route_capability_pin` names: a call only some providers serve + # drops the others, and a Decisions call carrying media skips the deployments refusing it. + handler: Final = plan.host_op.rsplit(".", 1)[-1] + decisions_media: Final = handler in DECISIONS_MEDIA_CALL_TYPES and bool( + request.get("audio") or request.get("videos") + ) + return { + **_request_narrowing(request), + **({"endpoint": endpoint} if endpoint is not None else {}), + **({"web_search": True} if web_search else {}), + **({"route_handler": handler} if serves_route_capability(handler) else {}), + **({"decisions_media": True} if decisions_media else {}), + } + + def _picks_async(self) -> bool: + return picks_async(normalize_strategy(self._resolved.routing_strategy)) + + def _process_renamed(self, model: str, group: str) -> str: + """``model`` as the error names it (``_common_checks_available_deployment``): the group a name + resolved to by name, alias or routing group, renamed through the process ``model_alias_map``.""" + resolution: Final = self._handle.resolve(model) + if resolution is None or resolution["kind"] not in ("Group", "Alias", "RoutingGroup"): + return model + renames: Final = self._host.dynamic().model_alias_map + return renames[group] if group in renames else group # noqa: SIM401 # `renames` is any process mapping, read as Python reads it + + def _max_parallel_requests_client(self, deployment: Mapping[str, object], kwargs: Mapping[str, object]) -> object: + """``_get_client(client_type="max_parallel_requests")``: the deployment's slot limit.""" + return max_parallel_requests_limit(self._port, deployment, kwargs) + + def _routable( + self, model: str, request: Mapping[str, object], call_type: str | None = None + ) -> Sequence[Mapping[str, object]]: + """``_core_get_routable_deployments``: ``call_type`` names the endpoint it asks about.""" + endpoint: Final = endpoint_for_call(call_type) + return self._handle.routable( + model, + _document( + { + **_start_fields(request), + **({"endpoint": endpoint} if endpoint is not None else {}), + "wildcard_request": wildcard_request(model), + } + ), + ) + + def _routing_fields(self, request: Mapping[str, object], messages: object = None) -> Mapping[str, object]: + """The request fields the Rust core routes on, and whether its tag stage may apply: the + router's ``enable_tag_filtering``, the request's own, or a deployment's + ``model_info.enable_tag_filtering`` (``get_deployments_for_tag``'s effective setting).""" + filter_tags: Final = ( + self._resolved.enable_tag_filtering + or request.get("enable_tag_filtering") is True + or self._model_tag_filtering() + ) + fields: dict[str, object] = dict(_start_fields(request)) # mutable-ok: built up below # rebind-ok: same + fields.update(self._group_policy_fields(request)) + if filter_tags: + fields["filter_tags"] = True + if request_carries_image(messages, request): + fields["has_image"] = True + return fields + + def _group_policies(self, request: Mapping[str, object]) -> tuple[object, GroupPolicies]: + """The ``model_group_retry_policy`` the retry loop reads for this request (the request's own, + even ``None``, replaces the router's: ``kwargs.pop(..., self.model_group_retry_policy)``), + resolved for every group name the call may look up.""" + policies: Final = request.get("model_group_retry_policy", self._resolved.model_group_retry_policy) + names: Final = self._policy_lookup_names(request) + resolved: Final = group_policies(policies, names) + if reads_as_policy(self._written_retry_policy): + return policies, resolved + # A group no entry names looks up the router's `retry_policy`, and that lookup raises. + unnamed: Final = names - frozenset(resolved.document) + return policies, dataclasses.replace(resolved, raising=resolved.raising | unnamed) + + def _policy_lookup_names(self, request: Mapping[str, object]) -> frozenset[str]: + """The model names a retry policy lookup may run on (each hop looks up the name it calls): + the requested one and every name in the fallback lists the walk may take.""" + resolved: Final = self._resolved + lists: Final = ( + resolved.fallbacks, + resolved.context_window_fallbacks, + resolved.content_policy_fallbacks, + *(request.get(key) for key in _REQUEST_FALLBACK_KEYS), + ) + return frozenset((*_strings(request.get("model")), *_strings(lists))) + + def _retry_policy_raises(self, request: Mapping[str, object]) -> Mapping[str, object]: + """The groups whose ``model_group_retry_policy`` lookup raises on a hop's first failure, for + router-core to end that hop with the error the lookup raises (see ``_Driver.take_events``).""" + raising: Final = raising_groups(request, self._group_policies(request)[1]) + return {"retry_policy_raises": list(raising)} if raising else {} + + def _group_policy_fields(self, request: Mapping[str, object]) -> Mapping[str, object]: + """The request's ``model_group_retry_policy`` for router-core; a router policy that is not a + mapping of policies rides on every request, resolved for it.""" + policies, resolved = self._group_policies(request) + if "model_group_retry_policy" not in request and is_policy_per_group(policies): + return {} + return {"model_group_retry_policy": _plain(resolved.document)} + + def _model_tag_filtering(self) -> bool: + """Whether any deployment sets ``model_info.enable_tag_filtering``, cached per registry version.""" + cached: Final = self._tag_filtering_models + if cached is not None and cached[0] == self._version: + return cached[1] + found: Final = any( + (as_str_mapping((as_str_mapping(record) or {}).get("model_info")) or {}).get("enable_tag_filtering") + is not None + for record in self._handle.deployments() + ) + self._tag_filtering_models = (self._version, found) + return found + + async def _drive( + self, + plan: Plan, + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + passthrough: "Callable[[Mapping[str, object], str], Awaitable[object]] | None" = None, + prepared_already: bool = False, + ) -> object: + routed: Final = ( + overrides if passthrough is None else {**(overrides or {}), "passthrough_on_no_deployment": True} + ) + # `owning_picks`: least-busy frees every pick this request made but never started. + with owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))): + call, prepared = self._start(plan, request, routed, prepared_already=prepared_already) + arm_compaction_state(self._port, plan, prepared) # pyright: ignore[reportArgumentType] # the call's own dict + with picks_reserve(plan.endpoint not in _UNRESERVED_OPS): + driver, step = await self._run(plan, call, prepared, fresh=True, passthrough=passthrough) + if step["kind"] == "failed": + raise driver.failed(step) + return await driver.finish_async(step) + + async def _run( + self, + plan: Plan, + call: "RouterCall[_CallSteps]", + prepared: Mapping[str, object], + cut_deployment: object = None, + stream_failure: tuple[int, BaseException] | None = None, + *, + fresh: bool = False, + passthrough: "Callable[[Mapping[str, object], str], Awaitable[object]] | None" = None, + ) -> "tuple[_Driver, Mapping[str, object]]": + """Drive ``call`` to its ``done`` or ``failed`` step, answering every step on the host. + ``cut_deployment`` is the attempt whose stream failed, for a recovery call, and + ``stream_failure`` the level it served with the failure a fallback walk re-enters from. + ``fresh`` is a call from the top, through the retry and fallback wrappers, and + ``passthrough`` serves an attempt whose pick failed.""" + driver: Final = _Driver( + self, plan, prepared, cut_deployment, stream_failure, fresh=fresh, passthrough=passthrough + ) + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + driver.take_events(step.get("events")) + # `run_async_fallback` stamps a served response before it logs the hop's success: `finish` + # runs what a served call still has pending. + if step["kind"] != "done": + await await_all(driver.outcomes.take_pending()) + match step["kind"]: + case "attempt": + outcome = await driver.attempt(step) + case "sleep": + await self._host.sleep(_sleep_seconds(step)) + outcome = None + case "allow_fallback": + outcome = await driver.allow_fallback(str(step["target"])) + case "refuse_fallbacks": + outcome = driver.refuse_fallbacks(step) + case "cache": + outcome = await self._answer_cache(step) + case "select": + outcome = await driver.select(step) + case "pre_route": + outcome = await driver.pre_route(step) + case "done" | "failed": + return driver, step + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + def _call_sync( + self, plan: Plan, request: Mapping[str, object], overrides: Mapping[str, object] | None = None + ) -> object: + # The sync routed calls pick with `picks_reserve(True)` inside `owning_picks`. + with owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))): + call, prepared = self._start(plan, request, overrides) + with picks_reserve(plan.endpoint not in _UNRESERVED_OPS): + driver, step = self._run_sync(plan, call, prepared, fresh=True) + if step["kind"] == "failed": + raise driver.failed(step) + return driver.finish(step) + + def _run_sync( + self, + plan: Plan, + call: "RouterCall[_CallSteps]", + prepared: Mapping[str, object], + cut_deployment: object = None, + stream_failure: tuple[int, BaseException] | None = None, + *, + fresh: bool = False, + ) -> "tuple[_Driver, Mapping[str, object]]": + driver: Final = _Driver(self, plan, prepared, cut_deployment, stream_failure, sync=True, fresh=fresh) + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + driver.take_events(step.get("events")) + if step["kind"] != "done": # see _run + driver.log_outcomes_sync() + match step["kind"]: + case "attempt": + outcome = driver.attempt_sync(step) + case "sleep": + self._host.sleep_sync(_sleep_seconds(step)) + outcome = None + case "allow_fallback": + from litellm.litellm_core_utils import asyncify + + outcome = asyncify.run_async_function(driver.allow_fallback, str(step["target"])) # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + case "refuse_fallbacks": + outcome = driver.refuse_fallbacks(step) + case "cache": + outcome = self._answer_cache_sync(step) + case "select": + outcome = driver.select_sync(step) + case "pre_route": + outcome = driver.pre_route_sync(step) + case "done" | "failed": + return driver, step + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + # -- stream recovery (the host's stream wrappers, router_rust.invoke.streaming) --------------------- + + def _recovery_call( + self, + recovery: str, + op: str, + request: Mapping[str, object], + origin: Mapping[str, object], + failure: BaseException, + ) -> "RouterCall[_CallSteps]": + start: Final = self._handle.retry_stream if recovery == "retry" else self._handle.fallback_stream + plan: Final = generic_plan(op) + routing: Final = _document( + { + "model": _routed_model(request), + **self._routing_fields(request), + **(self._selection_fields(plan, request) if plan is not None else {}), + **self._retry_policy_raises(request), + **continuation_routing(self, request), + **self._outcome_fields(plan, request), + } + ) + return start(op, routing, _document(origin), attempt_failure(failure)) + + def stream_policy( + self, request: Mapping[str, object], origin: Mapping[str, object], trigger: BaseException | None + ) -> Mapping[str, object]: + routing: Final = _document( + { + "model": _routed_model(request), + **_start_fields(request), + **self._group_policy_fields(request), + **self._retry_policy_raises(request), + } + ) + kind: Final = None if trigger is None else str(attempt_failure(trigger)["kind"]) + return self._handle.stream_policy(routing, _document(origin), kind) + + async def recover_stream( + self, + recovery: str, + op: str, + request: Mapping[str, object], + origin: Mapping[str, object], + failure: BaseException, + ) -> object: + """The fallback walk for a stream that failed before content: the fallback's response with the + router's headers, or the error the walk ended on.""" + call: Final = self._recovery_call(recovery, op, request, origin, failure) + driver, step = await self._run( + plan_for(op), call, request, origin.get("deployment_id"), (_origin_level(origin), failure) + ) + if step["kind"] == "failed": + raise driver.failed(step) + return await driver.finish_async(step) + + def recover_stream_sync( + self, + recovery: str, + op: str, + request: Mapping[str, object], + origin: Mapping[str, object], + failure: BaseException, + ) -> object: + call: Final = self._recovery_call(recovery, op, request, origin, failure) + driver, step = self._run_sync( + plan_for(op), call, request, origin.get("deployment_id"), (_origin_level(origin), failure) + ) + if step["kind"] == "failed": + raise driver.failed(step) + return driver.finish(step) + + async def retry_stream( + self, op: str, request: Mapping[str, object], origin: Mapping[str, object], failure: BaseException + ) -> "tuple[object, int, int] | BaseException": + """A same-group retry of a stream that failed before content: the retried response (without the + router's headers) with the retry count and budget, or the newest failure (``failure`` itself + when no retry ran).""" + call: Final = self._recovery_call("retry", op, request, origin, failure) + driver, step = await self._run(plan_for(op), call, request, origin.get("deployment_id")) + await await_all(driver.outcomes.take_pending()) + if step["kind"] == "failed": + return driver.last_error if driver.last_error is not None else failure + attempted: Final = step.get("attempted_retries") + budget: Final = step.get("max_retries") + return ( + driver.response, + attempted if isinstance(attempted, int) else 0, + budget if isinstance(budget, int) else 0, + ) + + def params_sent_to_deployment(self, request: Mapping[str, object], deployment_id: object) -> Mapping[str, object]: + """The request as a deployment received it: its params, the router defaults, its tool choice, then + the request (``_params_sent_to_deployment``).""" + record: Final = self._handle.deployment(deployment_id) if isinstance(deployment_id, str) else None + params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {} + defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None} + tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {} + return MappingProxyType({**params, **defaults, **tool_choice, **request}) + + def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]: + """Drive a call whose only steps are cache ops (``report_failure``) to its end.""" + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + match step["kind"]: + case "cache": + outcome = self._answer_cache_sync(step) + case "done": + return step + case "failed": + raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}") + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core asked for {other!r} while recording an outcome") + + async def _drive_cache_steps( + self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]" + ) -> Mapping[str, object]: + """``_drive_sync_steps`` answered through the async cache (``report_success``).""" + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + match step["kind"]: + case "cache": + outcome = await self._answer_cache(step) + case "done": + return step + case "failed": + raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}") + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core asked for {other!r} while recording an outcome") + + # -- pre-route steps ------------------------------------------------------------------------- + + async def _pre_route( + self, + step: Mapping[str, object], + request: Mapping[str, object], + messages: object = _NO_INPUT, + input: object = _NO_INPUT, + specific_deployment: object = _NO_INPUT, + *, + pass_through: bool = False, + ) -> object: + """The host's pre-routing of one pick (router_rust.steps.prerouting): the model group to resolve + instead of the step's ``model``, a custom strategy's deployment, or ``None``. ``pass_through`` + is ``_async_get_available_deployment_for_pass_through``, which runs the hook whatever the + router's strategy and which no replaced pick answers.""" + assigned: Final[object] = None if pass_through else self.__dict__.get("_core_async_get_available_deployment") + if callable(assigned): + # The async pick assigned through the deprecated attribute answers, as Python's + # replaced `_core_async_get_available_deployment` does. + answer: Final[object] = assigned( + model=str(step.get("model")), + request_kwargs=request, + messages=request.get("messages") if messages is _NO_INPUT else messages, + input=request.get("input") if input is _NO_INPUT else input, + specific_deployment=request.get("specific_deployment") + if specific_deployment is _NO_INPUT + else specific_deployment, + ) + chosen: Final[object] = await answer if inspect.isawaitable(answer) else answer + return _custom_pick(chosen) + custom: Final = None if pass_through else self._custom_strategy + if custom is not None: + picked: Final[object] = await custom.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick + model=str(step.get("model")), + messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + input=request.get("input") if input is _NO_INPUT else input, # pyright: ignore[reportArgumentType] # the request's input, as given + specific_deployment=request.get("specific_deployment") + if specific_deployment is _NO_INPUT + else specific_deployment, # pyright: ignore[reportArgumentType] # same + request_kwargs=request, # pyright: ignore[reportArgumentType] # the request kwargs, as given + ) + return _custom_pick(picked) + if not isinstance(request, dict): + return None + depth: Final = step.get("fallback_depth") + r: Final = self._resolved + owner: Final[object] = self.__dict__.get("_core_owner") + public: Final[object] = owner() if callable(owner) else None + return await self._prerouter.pre_route( + str(step.get("model")), + request, # pyright: ignore[reportUnknownArgumentType] # the request kwargs the hook stamps + messages=request.get("messages") if messages is _NO_INPUT else messages, + input=request.get("input") if input is _NO_INPUT else input, + fallback_depth=depth if isinstance(depth, int) else 0, + router_strategy=normalize_strategy(r.routing_strategy), + pass_through=pass_through, + enable_tag_filtering=r.enable_tag_filtering, + match_any=r.tag_filtering_match_any, + public_router=public if public is not None else self, + run_plugins=partial( + self._run_plugins, request, request.get("messages") if messages is _NO_INPUT else messages + ) # pyright: ignore[reportUnknownArgumentType] # the request kwargs the plugins stamp + if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list + else None, + ) + + async def _run_plugins( + self, + request: dict[str, object], # mutable-ok: the plugins stamp their outcome on the caller's request + messages: object, + model: str, + ) -> None: + """``_run_routing_plugins`` over the pre-routing hook's session-resolved ``model``.""" + await run_plugins( + tuple(self._hosted._core_routing_plugins), # pyright: ignore[reportPrivateUsage] # the live plugin list + self._hosted, # pyright: ignore[reportArgumentType] # the host registry serves the listing + self._resolved.model_group_alias, + model, + request, + messages, + ) + + def _custom_pick_sync( + self, + step: Mapping[str, object], + request: Mapping[str, object], + messages: object = _NO_INPUT, + input: object = _NO_INPUT, + specific_deployment: object = _NO_INPUT, + ) -> object: + """A custom strategy's sync pick (``get_available_deployment``); without one, no pre-routing.""" + custom: Final = self._custom_strategy + if custom is None: + return None + picked: Final[object] = custom.get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick + model=str(step.get("model")), + messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # see _pre_route + input=request.get("input") if input is _NO_INPUT else input, # pyright: ignore[reportArgumentType] # same + specific_deployment=request.get("specific_deployment") + if specific_deployment is _NO_INPUT + else specific_deployment, # pyright: ignore[reportArgumentType] # same + request_kwargs=request, # pyright: ignore[reportArgumentType] # same + ) + return _custom_pick(picked) + + # -- select steps ---------------------------------------------------------------------------- + + def _strategy_inputs(self, step: Mapping[str, object], request: Mapping[str, object]) -> "_StrategyInputs | None": + """The strategy, its selector, the name it keys state by and the candidate deployments of a + ``strategy`` select step; ``None`` for a stage the host passes through.""" + if step.get("stage") != "strategy": + return None + strategy: Final = str(step.get("strategy")) + group: Final = step.get("group") + selector: Final = self._selectors.selector_for( + strategy, str(step.get("scope")), str(group) if group is not None else None, request + ) + candidates: Final = [ + dict(as_str_mapping(self._handle.deployment(str(id))) or {}) for id in _as_list(step.get("candidates")) + ] + return strategy, selector, str(step.get("model")), candidates + + def _records_of(self, ids: object) -> list[dict[str, object]]: # mutable-ok: the deployment dicts filters take + return [dict(as_str_mapping(self._handle.deployment(str(id))) or {}) for id in _as_list(ids)] + + async def _answer_tags(self, step: Mapping[str, object], request: Mapping[str, object]) -> object: + """``get_deployments_for_tag`` over the step's candidates (bridged: not ported to Rust yet).""" + from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + + kept: Final = await get_deployments_for_tag( + ctx=RoutingContext(self), # pyright: ignore[reportArgumentType] # the filter reads registry() and settings() + model=str(step.get("model")), + healthy_deployments=self._records_of(step.get("candidates")), # pyright: ignore[reportArgumentType] # deployment dicts + request_kwargs=dict(request), # pyright: ignore[reportArgumentType] # the request kwargs + metadata_variable_name=get_metadata_variable_name_from_kwargs(dict(request)), + request_deployments=self._records_of(step.get("pool")), # pyright: ignore[reportArgumentType] # deployment dicts + ) + return [found for record in kept if (found := picked_id(record)) is not None] + + def _pre_call_facts( + self, step: Mapping[str, object], request: Mapping[str, object], messages: object, input: object + ) -> PreCallFacts: + hosted: Final = self._hosted + + def record(id: str) -> Mapping[str, object]: + return hosted._core_get_model_info(id) or as_str_mapping(self._handle.deployment(id)) or {} # pyright: ignore[reportPrivateUsage] # see __init__ + + return PreCallFacts(self._host, hosted, step, record, request, messages, input) # pyright: ignore[reportArgumentType] # the host registry serves the reads + + def _text_only(self, step: Mapping[str, object]) -> list[str]: # mutable-ok: the step's answer + """The candidates that declare no vision support (bridged: needs the cost map).""" + group: Final = str(step.get("model_group")) + return [ + str(id) + for id in _as_list(step.get("candidates")) + if declares_no_vision(as_str_mapping(self._handle.deployment(str(id))) or {}, group) + ] + + def _route_incapable(self, step: Mapping[str, object]) -> list[str]: # mutable-ok: the step's answer + """The candidates whose provider cannot serve the call the step's handler names.""" + group: Final = str(step.get("model_group")) + handler: Final = str(step.get("handler")) + return [ + str(id) + for id in _as_list(step.get("candidates")) + if cannot_serve_route(as_str_mapping(self._handle.deployment(str(id))) or {}, group, handler) + ] + + def _media_refusing( + self, step: Mapping[str, object], request: Mapping[str, object] + ) -> list[str]: # mutable-ok: the step's answer + """The candidates that refuse the audio or videos a Decisions call carries.""" + group: Final = str(step.get("model_group")) + defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None} + sent: Final = {**defaults, **request} + audio: Final = bool(request.get("audio")) + videos: Final = bool(request.get("videos")) + return [ + str(id) + for id in _as_list(step.get("candidates")) + if refuses_decisions_media( + as_str_mapping(self._handle.deployment(str(id))) or {}, group, sent, audio=audio, videos=videos + ) + ] + + async def _answer_select( + self, + step: Mapping[str, object], + request: Mapping[str, object], + messages: object, + input: object, + *, + pass_through: bool = False, + ) -> object: + if step.get("stage") == "pre_call": + return await self._pre_call_facts(step, request, messages, input).answer_async() + if step.get("stage") == "routing_plugins": + # The plugins already ran in this pick's pre_route step (see `_pre_route`), on the + # session-resolved name and before a strategy was selected, as Python's hook runs them. + return candidate_models(request) + if step.get("stage") == "tags": + return await self._answer_tags(step, request) + if step.get("stage") == "text_only": + return self._text_only(step) + if step.get("stage") == "route_incapable": + return self._route_incapable(step) + if step.get("stage") == "media_refusing": + return self._media_refusing(step, request) + if step.get("stage") == "provider_request": + return _provider_request(step) + inputs: Final = self._strategy_inputs(step, request) + if inputs is None: + return step["candidates"] + strategy, selector, model, candidates = inputs + pinned: Final = bool(request.get("_encrypted_content_affinity_pinned")) and len(candidates) == 1 + if selector is None and not pinned: + return [] + # The strategy's service event: `_async_get_available_deployment` times it for the strategies + # with an async pick, `_async_get_available_deployment_for_pass_through` for any, on another clock. + timed: Final = not pinned and strategy != "simple-shuffle" and (pass_through or self._picks_async()) + clock: Final = perf_counter if pass_through else unix_time + start_time: Final = clock() + # `_async_get_available_deployment`: a pin by encrypted content takes no strategy's pick. + picked: Final = ( + candidates[0] + if pinned + else await pick_async( + strategy, + selector, # pyright: ignore[reportArgumentType] # a selector from selector_for + model, + candidates, + messages, + input, + dict(request), + # `_async_get_available_deployment` reserves a request's pick unless told not to; + # the pass-through pick only when told to. + reserve=pick_reserves(default=not pass_through) and bool(request), + ) + ) + if picked is not None and selector is not None and self._selectors.is_override(strategy, selector): + await selector.async_pre_call_check(dict(as_str_mapping(picked) or {}), None) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # every selector is a CustomLogger + if picked is not None and timed: + self.service_logged(".async_get_available_deployments", start_time, clock(), request) + found: Final = picked_id(picked) + return [found] if found is not None else [] + + def _answer_select_sync( + self, step: Mapping[str, object], request: Mapping[str, object], messages: object, input: object + ) -> object: + if step.get("stage") == "pre_call": + return self._pre_call_facts(step, request, messages, input).answer() + if step.get("stage") == "routing_plugins": + return candidate_models(request) + if step.get("stage") == "callback_filters": + # Only the async candidate lists run the callback filters. + return step["candidates"] + if step.get("stage") == "text_only": + return self._text_only(step) + if step.get("stage") == "route_incapable": + return self._route_incapable(step) + if step.get("stage") == "media_refusing": + return self._media_refusing(step, request) + if step.get("stage") == "provider_request": + return _provider_request(step) + inputs: Final = self._strategy_inputs(step, request) + if inputs is None: + return step["candidates"] + strategy, selector, model, candidates = inputs + if selector is None: + return [] + picked: Final = pick_sync( + strategy, + selector, # pyright: ignore[reportArgumentType] # a selector from selector_for + model, + candidates, + messages, + input, + dict(request), + # The sync picks reserve only when a routed call site says so (`picks_reserve(True)`). + reserve=pick_reserves(default=False) and bool(request), + ) + if picked is not None and self._selectors.is_override(strategy, selector): + selector.pre_call_check(dict(as_str_mapping(picked) or {})) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # see above + found: Final = picked_id(picked) + return [found] if found is not None else [] + + # -- cache steps ----------------------------------------------------------------------------- + + def _store_for(self, key: str) -> "DualCache": + """Cooldowns live in the cooldown store; every other routing key in the routing cache.""" + return self._cooldowns.cooldown_store if key.endswith(":cooldown") else self._cache + + async def _batch_read(self, keys: list[str]) -> Sequence[object]: # mutable-ok: the cache's key list + """One batch read, as the Python core makes it: the keys memory cannot serve, at most once per + read interval. Cooldowns are ``async_get_cooldown_deployments``' read, through the request's + armed prefetch when there is one.""" + if not keys or not keys[0].endswith(":cooldown"): + read: Final[object] = await self._cache.async_batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return read if isinstance(read, list) else [None] * len(keys) # pyright: ignore[reportUnknownVariableType] # see above + store: Final = self._cooldowns.cooldown_store + with service_target(ROUTER_COOLDOWNS_TARGET): + prefetched: Final = await RoutingReadBatch._read_prefetched(((store, keys),)) # pyright: ignore[reportPrivateUsage] # the batch's own prefetch read + results: Final[object] = ( + prefetched[0] if prefetched is not None else await store.async_batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType] # see above + ) + return self._corrected_cooldowns(keys, results) + + def _batch_read_sync(self, keys: list[str]) -> Sequence[object]: # mutable-ok: the cache's key list + """``_batch_read`` for a sync pick: ``get_active_cooldowns``' read.""" + if not keys or not keys[0].endswith(":cooldown"): + read: Final[object] = self._cache.batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return read if isinstance(read, list) else [None] * len(keys) # pyright: ignore[reportUnknownVariableType] # see above + with service_target(ROUTER_COOLDOWNS_TARGET): + results: Final = as_object_list(self._cooldowns.cooldown_store.batch_get_cache(keys=keys)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # see above + return self._corrected_cooldowns(keys, results) + + def _corrected_cooldowns(self, keys: Sequence[str], results: object) -> Sequence[object]: + """The cooldowns a batch read found, once ``active_cooldowns_from_results`` corrected the in-memory + TTL of each it promoted from Redis and dropped each that expired.""" + if not isinstance(results, list): + return [None] * len(keys) + ids: Final = [key.removeprefix("deployment:").removesuffix(":cooldown") for key in keys] + _ = self._cooldowns.active_cooldowns_from_results(ids, results) # pyright: ignore[reportUnknownArgumentType] # see above + return results # pyright: ignore[reportUnknownVariableType] # see above + + async def _increment_usage(self, keys: Sequence[object], amounts: Sequence[object], ttl: float | None) -> None: + """``_increment_deployment_usage``'s write: one pipeline, then this worker's memory holds what + Redis answered, so its usage reads see what the other workers counted.""" + from litellm.types.caching import RedisPipelineIncrementOperation + + operations: Final = [ + RedisPipelineIncrementOperation( + key=str(key), increment_value=_amount(amount), ttl=None if ttl is None else int(ttl) + ) + for key, amount in zip(keys, amounts) + ] + values: Final = await self._cache.async_increment_cache_pipeline(increment_list=operations) + if values is None or self._cache.redis_cache is None: + return + for operation, value in zip(operations, values): + await self._cache.async_set_cache(operation["key"], int(value), local_only=True, ttl=operation["ttl"]) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + + async def _answer_cache(self, step: Mapping[str, object]) -> object: + with service_target(_cache_target(str(step.get("key")))): + return await self._answer_cache_in_target(step) + + async def _answer_cache_in_target(self, step: Mapping[str, object]) -> object: + key: Final = str(step.get("key")) + ttl: Final = _ttl(step) + local: Final = step.get("local") is True + match step.get("op"): + case "get": + return _plain(await self._store_for(key).async_get_cache(key, local_only=local)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # DualCache's reads are untyped + case "batch_get": + return _plain(await self._batch_read([str(k) for k in _as_list(step.get("keys"))])) + case "set": + store: Final = self._store_for(key) + if ttl is None: + await store.async_set_cache(key, step.get("value")) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + else: + await store.async_set_cache(key, step.get("value"), ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + return None + case "increment": + by: Final = _amount(step.get("value")) + counter: Final = self._store_for(key) + total: Final = ( + await counter.async_increment_cache(key, by, local_only=local) # pyright: ignore[reportUnknownMemberType] # see above + if ttl is None + else await counter.async_increment_cache(key, by, local_only=local, ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + ) + return by if total is None else total + case "increment_usage": + await self._increment_usage(_as_list(step.get("keys")), _as_list(step.get("value")), ttl) + return None + case "delete": + await self._store_for(key).async_delete_cache(key) + return None + case other: + raise RuntimeError(f"router-core asked for an unknown cache op {other!r}") + + def run_routing_cache_sync(self, read_or_write: "Callable[[], Coroutine[object, object, object]]") -> object: + """A routing-cache coroutine on a sync call, whose `make_call` runs on the one event loop + `run_async_function` opened for the whole call. Without Redis the coroutine never suspends, so + it runs here: a loop of its own would hand the GIL to the logging thread, which then reads the + response before the router's headers are on it.""" + if self._cache.redis_cache is not None: + from litellm.litellm_core_utils import asyncify + + return asyncify.run_async_function(read_or_write) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the untyped sync bridge + running: Final = read_or_write() + try: + while True: + _ = running.send(None) + except StopIteration as finished: + return finished.value # pyright: ignore[reportAny] # the coroutine's result + + def _answer_cache_sync(self, step: Mapping[str, object]) -> object: + with service_target(_cache_target(str(step.get("key")))): + return self._answer_cache_sync_in_target(step) + + def _answer_cache_sync_in_target(self, step: Mapping[str, object]) -> object: + key: Final = str(step.get("key")) + ttl: Final = _ttl(step) + local: Final = step.get("local") is True + match step.get("op"): + case "get": + return _plain(self._store_for(key).get_cache(key, local_only=local)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # DualCache's reads are untyped + case "batch_get": + return _plain(self._batch_read_sync([str(k) for k in _as_list(step.get("keys"))])) + case "set": + store: Final = self._store_for(key) + if ttl is None: + store.set_cache(key, step.get("value")) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + else: + store.set_cache(key, step.get("value"), ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + return None + case "increment": + by: Final = _amount(step.get("value")) + counter: Final = self._store_for(key) + # DualCache.increment_cache is annotated int but adds floats as given. + return ( + counter.increment_cache(key, by, local_only=local) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # see above + if ttl is None + else counter.increment_cache(key, by, local_only=local, ttl=ttl) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # see above + ) + case "increment_usage": + keys: Final = _as_list(step.get("keys")) + amounts: Final = _as_list(step.get("value")) + self.run_routing_cache_sync(lambda: self._increment_usage(keys, amounts, ttl)) + return None + case "delete": + self._store_for(key).delete_cache(key) # pyright: ignore[reportUnknownMemberType] # DualCache's delete is untyped + return None + case other: + raise RuntimeError(f"router-core asked for an unknown cache op {other!r}") + + # -- errors the core raises when no attempt failed ------------------------------------------ + + def _failure_error( # noqa: C901 # one branch per failure Python tells apart + self, + step: Mapping[str, object], + model: str, + last: BaseException | None, + *, + pass_through: str = "", + request: Mapping[str, object] | None = None, + ) -> BaseException: + """The error a call or pick that failed before any attempt raises; ``pass_through`` names + the selection kind (``pass_through``/``async_pass_through``) whose errors it words.""" + import litellm.exceptions as litellm_exceptions + + kind: Final = step.get("error_kind") + aliases: Final = self._resolved.model_group_alias + refused: Final = selection_refusal(step, model, request or {}) + if refused is not None: + return refused + if kind == "paused": + # The message names the deployment's litellm model, not the id that was asked for; the + # default deployment's copy is named by the requested name. + record: Final = as_str_mapping(self._handle.deployment(model)) + litellm_model: Final = ( + str((as_str_mapping(record.get("litellm_params")) or {}).get("model")) if record is not None else model + ) + return litellm_exceptions.ServiceUnavailableError( + message=f"Model '{litellm_model}' is currently paused and cannot accept requests.", + model=litellm_model, + llm_provider="", + ) + if kind == "only_markers": + named_model: Final = str(step.get("error_model") or model) + return litellm_exceptions.BadRequestError( + message=f"You passed in model={named_model}. {RouterErrors.only_strategy_marker_deployments.value}", + model=named_model, + llm_provider="", + ) + if kind == "endpoint_not_supported": + return litellm_exceptions.EndpointNotSupportedByModelError( + message=str(step.get("error_detail")), + model=str(step.get("error_model") or model), + llm_provider="", + ) + if kind == "ptu_reserved": + reserved: Final = str(step.get("error_model") or model) + return litellm_exceptions.BadRequestError( + message=f"Deployment {reserved} is reserved for the teams holding a PTU share of it", + model=reserved, + llm_provider="", + ) + if kind == "not_pass_through": + # The sync pick names what `_common_checks_available_deployment` resolved the id to. + named_record: Final = ( + as_str_mapping(self._handle.deployment(model)) if pass_through == "pass_through" else None + ) + error_model: Final = ( + str((as_str_mapping(named_record.get("litellm_params")) or {}).get("model")) + if named_record is not None + else model + ) + return litellm_exceptions.BadRequestError( + message=f"Deployment {model} does not support pass-through endpoint (use_in_pass_through=False)", + model=error_model, + llm_provider="", + ) + if kind == "no_pass_through_deployments": + # The sync pick names what `_common_checks_available_deployment` resolved the name to. + named_group: Final = str(step.get("error_model") or model) + listed_model: Final = ( + named_group if pass_through == "async_pass_through" else self._process_renamed(model, named_group) + ) + message: Final = ( + f"Model {listed_model} has no deployments configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true to the deployment configuration" + if pass_through == "async_pass_through" + else f"Model {listed_model} has no deployment configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true in the deployment configuration" + ) + return litellm_exceptions.BadRequestError(message=message, model=listed_model, llm_provider="") + if kind == "context_window_exceeded": + return litellm_exceptions.ContextWindowExceededError( + message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context " + f"window large enough for this call.\n{step.get('error_detail')}", + model=str(step.get("error_model") or model), + llm_provider="", + ) + if kind == "unsupported_params": + params: Final = sorted(str(param) for param in _as_list(step.get("error_params"))) + unsupported_model: Final = str(step.get("error_model") or model) + return litellm_exceptions.UnsupportedParamsError( + message=( + f"no deployment of model={unsupported_model} supports {params}. " + "To drop these, set `litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, " + f"or send allowed_openai_params={params} to pass them through" + ), + model=unsupported_model, + ) + if kind == "routing_plugins_left_nothing": + left: Final = [str(name) for name in _as_list(step.get("error_params"))] + return ValueError(f"No deployments left after routing-plugin filtering. candidate_models={left}") + if kind == "routing_plugins_need_async": + return ValueError(SYNC_PATH_MESSAGE) + looked_up: Final = _lookup_error(kind, str(step.get("error_model") or model), self._resolved.default_deployment) + if looked_up is not None: + return looked_up + if kind == "unknown_model": + # router-core names what `_common_checks_available_deployment` looked for last. + aliased: Final = _alias_target(aliases.get(model)) or model if model in aliases else model + named: Final = str(step.get("error_model") or aliased) + return litellm_exceptions.BadRequestError( + message=f"You passed in model={named}. {RouterErrors.no_healthy_deployments.value}", + model=named, + llm_provider="", + ) + if kind == "no_healthy_deployments": + group: Final = self.registry().resolve(model) + group_ids: Final = self.registry().names(str(group), kind="ids") + # A listed pass-through pick names only the pass-through deployments. + ids: Final = ( + [id for id in group_ids if _uses_pass_through(self._handle.deployment(id))] + if pass_through == "pass_through" + else list(group_ids) + ) + active: Final = self._cooldowns.get_active_cooldowns(list(ids), None) # pyright: ignore[reportArgumentType] # a list of ids + # `async_get_cooldown_deployments_with_debug_info`: every deployment cooling, in any group. + listed: Final = self._cooldowns.get_active_cooldowns(list(self.registry().names(kind="ids")), None) # pyright: ignore[reportArgumentType] # a list of ids + cooling: Final = [id for id, _ in listed] + min_cooldown: Final = min( + (_seconds(entry.get("cooldown_time"), self._cooldowns.default_cooldown_time) for _, entry in active), + default=_seconds(None, self._cooldowns.default_cooldown_time), + ) + return RouterRateLimitError( + model=self._process_renamed(model, str(group)), + cooldown_time=min_cooldown, + enable_pre_call_checks=self._resolved.enable_pre_call_checks, + cooldown_list=cooling, + model_ids=list(ids), # pyright: ignore[reportArgumentType] # a list of ids + cooldown_deployments=get_cooldown_deployment_info( + RoutingContext(self), # pyright: ignore[reportArgumentType] # reads registry() + ids, + cooling, + ), + ) + if last is not None: + return last + return RuntimeError(f"router-core ({kind}): {step.get('error')}") + + +def wildcard_request(model: str) -> str: + """``f"{provider}/{model}"``, the spelling ``PatternMatchRouter.get_pattern`` tries second.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return f"{inferred_provider(model)}/{model}" + + +def _provider_request(step: Mapping[str, object]) -> Mapping[str, object]: + return {"models": [wildcard_request(str(step.get("model")))]} + + +def _lookup_error(kind: object, model: str, default_deployment: object) -> BaseException | None: + """The plain Python error a pick's lookup raised for ``model``, or None for any other failure.""" + if kind == "default_deployment_raises": + return default_deployment_copy_error(default_deployment, model) + if kind == "index_out_of_range": + # `self._core_model_list[index]` for a written index position past the list. + return IndexError("list index out of range") + if kind == "missing_routing_group": + # `_get_routing_context` reads the stale member map's group from the replaced groups. + return KeyError(model) + return None + + +def _origin_level(origin: Mapping[str, object]) -> int: + level: Final = origin.get("level") + return level if isinstance(level, int) else 0 + + +def _sleep_seconds(step: Mapping[str, object]) -> float: + """A retry's sleep: ``_time_to_sleep_before_retry`` returns a plain ``0`` when another + deployment is healthy, else a backoff in float seconds.""" + seconds: Final = float(str(step["seconds"])) + return 0 if seconds == 0 else seconds + + +def _uses_pass_through(record: object) -> bool: + params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {} + return bool(params.get("use_in_pass_through")) + + +def _seconds(value: object, default: float) -> float | int: + """A cooldown time as stored: an int stays an int, so error texts read ``60`` like Python's.""" + if isinstance(value, int | float) and not isinstance(value, bool): + return value + return int(default) if float(default).is_integer() else default + + +def _ttl(step: Mapping[str, object]) -> float | None: + ttl: Final = step.get("ttl") + return ttl if isinstance(ttl, int | float) else None + + +def _amount(value: object) -> float | int: + return value if isinstance(value, int | float) and not isinstance(value, bool) else 1 + + +# The plans whose kwargs carry `original_generic_function`, which the fallback cooldown reads. +_GENERIC_API_KINDS: Final = frozenset(("generic", "search")) + + +class _NotSynchronous(TypeError): + """A sync attempt's host endpoint returned an awaitable: a host bug, never an attempt failure.""" + + +_MEMBER_READ_PICKS: Final = frozenset({"healthy", "pass_through", "sync_pick"}) + + +class _Selection: + """One ``select``: the Rust selection call, its select and cache steps answered on the host.""" + + def __init__( + self, + core: RustCore, + model: object, + request: dict[str, object], # mutable-ok: the caller's request, whose narrowing keys are popped + kind: str, + specific_deployment: bool | None, + messages: object, + input: object, + *, + probe: bool = False, + ) -> None: + self._core: Final = core + # `_async_get_available_deployment` reads the alias map first, every other pick its names. + self._model: Final = ( + model if isinstance(model, str) else core.routed_name(model, read_as_member=kind in _MEMBER_READ_PICKS) + ) + self._request: Final = request + self._kind: Final = kind + self._messages: Final = messages + self._input: Final = input + fields: Final = { + **core._routing_fields(request, messages), # pyright: ignore[reportPrivateUsage] # the selection serves its own core + **_narrowing_fields(request, kind), + } # pyright: ignore[reportPrivateUsage] # the selection serves its own core + # The pick's argument alone: a `specific_deployment` key in its request is no argument, and + # `_common_checks_available_deployment` reads nothing but True. + fields.pop("specific_deployment", None) + if specific_deployment is True: + fields["specific_deployment"] = True + # `filter_web_search_deployments` is in the async chain, which a strategy without an async + # selector never reaches on a pick. + if (kind in ("healthy", "async_pass_through") or (kind == "pick" and core._picks_async())) and ( # pyright: ignore[reportPrivateUsage] # the selection serves its own core + is_web_search_request(request) + ): + fields["web_search"] = True + # `_text_completion` wraps its sync pick in `endpoint_selection`. + endpoint: Final = selected_endpoint() + if endpoint is not None: + fields["endpoint"] = endpoint + if probe: + fields["health_check_probe"] = True + if kind in ("pick", "async_pass_through") or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core + fields["pre_routes"] = True + self._specific_deployment: Final = specific_deployment + # The error a callback filter raised, which a failed selection raises. + self._filter_error: BaseException | None = None + if kind == "pass_through": + fields["pass_through"] = "listed" + elif kind == "async_pass_through": + fields["pass_through"] = "healthy" + core._sync_routing_state() # pyright: ignore[reportPrivateUsage] # see above + handle: Final = core._handle # pyright: ignore[reportPrivateUsage] # see above + self._call: Final = (handle.healthy if kind == "healthy" else handle.select)(self._model, _document(fields)) + + def _done(self, step: Mapping[str, object]) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + if self._kind == "healthy": + if step.get("single"): + # A deployment named by its id is listed alone, as a dict. + (record,) = _as_list(step.get("deployments")) + return for_requested_model(typed_deployment(as_str_mapping(record) or {}), self._model) + # The deployments as served: a priority group's members carry their priority as order. + return [ + for_requested_model(typed_deployment(as_str_mapping(record) or {}), self._model) + for record in _as_list(step.get("deployments")) + ] + if step.get("deployment_id") in (DEFAULT_DEPLOYMENT_FAILING_ID, DEFAULT_DEPLOYMENT_COPIED_ID): + # A default deployment every call to fails, or whose id is not a string, is picked as Python copies it. + picked: Final = default_deployment_copy(self._core._resolved.default_deployment, self._model) # pyright: ignore[reportPrivateUsage] # the selection serves its own core + return picked # pyright: ignore[reportReturnType] # the assigned value's copy, as the Python core returns it + return for_requested_model(typed_deployment(as_str_mapping(step.get("deployment")) or {}), self._model) + + def _failed(self, step: Mapping[str, object]) -> BaseException: + core: Final = self._core + error: Final = core._failure_error( # pyright: ignore[reportPrivateUsage] # see above + step, self._model, self._filter_error, pass_through=self._kind, request=self._request + ) + if self._kind not in ("pass_through", "sync_pick"): + _log_refused_pick(self._request, error) + return error + + async def run(self) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + core: Final = self._core + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = self._call.next(outcome) + match step["kind"]: + case "cache": + outcome = await core._answer_cache(step) # pyright: ignore[reportPrivateUsage] # see above + case "select" if step.get("stage") == "callback_filters": + outcome, self._filter_error = await core._answer_callback_filters( # pyright: ignore[reportPrivateUsage] # see above + step, self._request, self._messages + ) + case "select": + outcome = await core._answer_select( # pyright: ignore[reportPrivateUsage] # see above + step, + self._request, + self._messages, + self._input, + pass_through=self._kind == "async_pass_through", + ) + case "pre_route": + outcome = await core._pre_route( # pyright: ignore[reportPrivateUsage] # see above + step, + self._request, + self._messages, + self._input, + self._specific_deployment, + pass_through=self._kind == "async_pass_through", + ) + case "done": + done = self._done(step) + if self._kind != "pick" or not isinstance(done, Mapping): + return done + # `_core_async_get_available_deployment` resolves its pick to a pod. + return await self._resolved_pod(core, done) + case "failed": + raise self._failed(step) + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + async def _resolved_pod(self, core: "RustCore", done: Mapping[str, object]) -> Mapping[str, object]: + discovery: Final = core._pick_discovery(sync=False) # pyright: ignore[reportPrivateUsage] # the selection serves its own core + if discovery is None: + return done + resolved: Final[object] = await discovery.async_resolve_deployment(done, self._request) + return resolved # pyright: ignore[reportReturnType] # the deployment, resolved + + def _resolved_pod_sync(self, core: "RustCore", done: Mapping[str, object]) -> Mapping[str, object]: + discovery: Final = core._pick_discovery(sync=True) # pyright: ignore[reportPrivateUsage] # see _resolved_pod + return done if discovery is None else discovery.resolve_deployment(done, self._request) + + def run_sync(self) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + core: Final = self._core + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = self._call.next(outcome) + match step["kind"]: + case "cache": + outcome = core._answer_cache_sync(step) # pyright: ignore[reportPrivateUsage] # see above + case "select": + outcome = core._answer_select_sync(step, self._request, self._messages, self._input) # pyright: ignore[reportPrivateUsage] # see above + case "pre_route": + outcome = core._custom_pick_sync( # pyright: ignore[reportPrivateUsage] # see above + step, self._request, self._messages, self._input, self._specific_deployment + ) + case "done": + done = self._done(step) + if self._kind != "sync_pick" or not isinstance(done, Mapping): + return done + # `get_available_deployment` resolves its pick to a pod. + return self._resolved_pod_sync(core, done) + case "failed": + raise self._failed(step) + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + +def _log_refused_pick(request: Mapping[str, object], error: BaseException) -> None: + """A refused pick is logged to the request's logging object (`_async_get_available_deployment`).""" + dispatch: Final[object] = getattr(request.get("litellm_logging_obj"), "dispatch_failure_handlers", None) + if not callable(dispatch): + return + import asyncio + import traceback + + trace: Final = "Traceback (most recent call last):\n" + "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + pending: Final[object] = dispatch(exception=error, traceback_exception=trace, prefer_async_handlers=True) + if inspect.isawaitable(pending): + _ = asyncio.ensure_future(pending) + + +def _log_refused_image_pick(request: Mapping[str, object], error: BaseException) -> None: + """``_aimage_generation``'s refused pick, for a request with no logging object of its own.""" + from litellm._uuid import uuid + from litellm.router_rust.invoke.endpoints import ROUTING_KEYS + from litellm.utils import ( + Rules, + _restore_correlation_context_if_supported, # pyright: ignore[reportPrivateUsage] # same task-local cleanup as the Python router + function_setup, + ) + + kwargs: Final = without(request, *(ROUTING_KEYS - {"model"})) + kwargs.setdefault("litellm_call_id", str(uuid.uuid4())) + logging_obj, _ = function_setup("aimage_generation", Rules(), datetime.now(), **kwargs) # noqa: DTZ005 # SDK logging uses naive local timestamps # pyright: ignore[reportArgumentType] # opaque SDK options, as the Python router passes them + try: + _log_refused_pick({"litellm_logging_obj": logging_obj}, error) + finally: + _restore_correlation_context_if_supported(logging_obj) + + +def _custom_pick(picked: object) -> Mapping[str, object]: + """A custom strategy's answer as the deployment document the Rust core serves.""" + document: Final = as_str_mapping(picked) + if document is None: + raise TypeError(f"the custom routing strategy returned {type(picked).__name__}, not a deployment") + return _document(document) + + +def _with_target_order(kwargs: Mapping[str, object], order: object) -> Mapping[str, object]: + """The attempt's kwargs with the order level its hop targets, as `run_async_fallback` sets + `_target_order` before `_acreate_file` picks every healthy deployment of that level.""" + if not isinstance(order, int) or isinstance(order, bool): + return kwargs + return {**kwargs, "_target_order": order} + + +def _narrowing_fields(request: dict[str, object], kind: str) -> Mapping[str, object]: # mutable-ok: popped + """``_target_order``, ``_excluded_deployment_ids`` and ``_retry_skipped_deployment_ids``, popped + from the request as the healthy chain pops them (a listed pass-through pick only reads the order).""" + fields: Final = _request_narrowing(request) + if kind == "pass_through": + return {key: value for key, value in fields.items() if key == "target_order"} + for key in ("_target_order", "_excluded_deployment_ids", "_retry_skipped_deployment_ids"): + request.pop(key, None) + return fields + + +def _request_narrowing(request: Mapping[str, object]) -> Mapping[str, object]: + """The narrowing keys a request carries, as the fields router-core reads them.""" + from litellm.litellm_core_utils.routing_common_utils import ( + as_excluded_deployment_ids, + as_retry_skipped_deployment_ids, + ) + + target_order: Final = request.get("_target_order") + excluded: Final = as_excluded_deployment_ids(request.get("_excluded_deployment_ids")) + skipped: Final = as_retry_skipped_deployment_ids(request.get("_retry_skipped_deployment_ids")) + return { + **( + {"target_order": target_order} + if isinstance(target_order, int) and not isinstance(target_order, bool) + else {} + ), + **({"excluded_ids": list(excluded)} if excluded else {}), + **({"retry_skipped_ids": list(skipped)} if skipped else {}), + } + + +_NO_ROUTING_PRE_CALL_CHECKS: Final = frozenset({"search", "rerank", "moderation"}) +# Attempts whose deployment is no pick (`_core_async_get_available_deployment` / +# `get_available_deployment`), so Kubernetes pod discovery never resolves it. +_UNPICKED_KINDS: Final = frozenset({"search", "create_file"}) + + +class _Driver: + """Answers the attempt and fallback steps of one call, and shapes its result. + + Each attempt is one host call against the deployment Rust picked. The driver keeps the error + each fallback level ended on: a level that fails raises its own error, whatever its fallbacks + raised, as the Python router does, unless an order chain handed it another + (``order_chain_failed``). + """ + + def __init__( + self, + core: RustCore, + plan: Plan, + request: Mapping[str, object], + cut_deployment: object = None, + stream_failure: tuple[int, BaseException] | None = None, + *, + sync: bool = False, + fresh: bool = False, + passthrough: "Callable[[Mapping[str, object], str], Awaitable[object]] | None" = None, + ) -> None: + self._core: Final = core + self._sync: Final = sync + self._passthrough: Final = passthrough + self._plan: Final = plan + self._op: Final = plan.endpoint + self._request: Final = request + self._cut_deployment: Final = cut_deployment if isinstance(cut_deployment, str) else None + self._last_response: object = None + self._last_error: BaseException | None = None + # The level the call raises from: a fallback walk re-entered from a failed stream starts at + # the level that stream served, on the failure ``async_function_with_fallbacks_common_utils`` + # was handed, which is what it raises when no fallback answers. + self._base_level: Final = stream_failure[0] if stream_failure is not None else 0 + self._level_errors: Final[dict[int, BaseException]] = dict( # mutable-ok: filled as the call runs + (stream_failure,) if stream_failure else () + ) + # The retry counts an exhausted retry loop stamps on the pick error its level then raises. + self._pending_stamps: Final[dict[int, tuple[object, object]]] = {} # mutable-ok: filled as the call runs + # Each level's current hop's first failure, once it retries: the error its retry loop raises + # last is raised while that one is handled, so it chains to it. + self._hop_first: Final[dict[int, BaseException]] = {} # mutable-ok: filled as the call runs + # Each level's latest pick error: the one object it logged, recorded and raises. + self._level_picks: Final[dict[int, BaseException]] = {} # mutable-ok: filled as the call runs + self._served: tuple[str | None, Mapping[str, object]] = (None, MappingProxyType({})) + # The latest attempt's deployment and call, which a blocked response is raised from. + self._latest_call: tuple[Mapping[str, object], AttemptCall] | None = None + # A sync attempt's stream is reading its first chunk, so a failure now is the stream's to log. + self._reading_first_chunk: bool = False + self._attempts: int = 0 + # When the latest attempt started: Python reads it off the logging object, when one is set up. + self._attempt_started: datetime | None = None + # A single-attempt endpoint runs neither of the wrappers that stamp the walk on the request. + self._walk: Final = WalkMetadata( + request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt + ) + self.outcomes: Final = FallbackOutcomes(request) + # The fallback level the walk is at, from its fallback events. + self._level: int = self._base_level + # The primary level reached the end of `async_function_with_fallbacks_common_utils`, the + # only place the outcome message is appended: not its early raises, nor its order path. + self._explains_failure: bool = False + + def _blocked(self, response: object, endpoint_name: str) -> str | None: + """How the provider blocked a served response, the first check of + ``_should_raise_content_policy_error`` (a chat response with no choices, or whose first + choice was content-filtered) and of ``_should_raise_anthropic_refusal_error`` (a + /v1/messages safeguard refusal carrying ``stop_details``).""" + from litellm.llms.anthropic.messages.native.utils import get_safeguard_refusal_stop_details + from litellm.types.utils import ModelResponse + + if endpoint_name == "anthropic_messages" and get_safeguard_refusal_stop_details(response) is not None: + return "refusal" + if self._op not in ("acompletion", "completion") or not isinstance(response, ModelResponse): + return None + return ( + "content_filter" if not response.choices or response.choices[0].finish_reason == "content_filter" else None + ) + + def _blocked_error(self, event: Mapping[str, object]) -> BaseException: + """The error the router raises for a blocked response a fallback would serve.""" + import litellm.exceptions as litellm_exceptions + from litellm.llms.anthropic.messages.native.utils import ( + get_safeguard_refusal_stop_details, + safeguard_refusal_error, + ) + + model: Final = str(event.get("model")) + stop_details: Final = get_safeguard_refusal_stop_details(self._last_response) + if event.get("blocked") == "refusal" and stop_details is not None: + return safeguard_refusal_error(model=model, stop_details=stop_details) + return litellm_exceptions.ContentPolicyViolationError( + message="Response output was blocked.", model=model, llm_provider="" + ) + + def _stamp_blocked(self, error: BaseException) -> None: + """The attempt helper's ``except`` runs for the error it raises over a blocked response: the + response then counts as a failure, not a success.""" + if self._latest_call is None: + return + deployment, call = self._latest_call + endpoints: Final = self._core._endpoints # pyright: ignore[reportPrivateUsage] # the driver serves its own core + stamp_error(self._plan, error, deployment, call.served) + endpoints.counters.retract_success(self._plan, call) + endpoints.attempt_failed(self._plan, error, deployment, call, call.group) + + def _params_base_model(self, deployment_id: str) -> str | None: + """``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``.""" + record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core + base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model") + return base_model if isinstance(base_model, str) else None + + def _logged_error(self, event: Mapping[str, object]) -> BaseException | None: + """The error a ``retry_logged`` event records.""" + return self._cause_error(event) + + def _policy_raised(self, event: Mapping[str, object]) -> None: + """A hop's first failure looked its group's retry policy up and the lookup raised: the hop + ends with that error, raised from the failure the event names.""" + model: Final = str(event.get("model")) + original: Final = self._cause_error(event) or RuntimeError(f"router-core: the hop serving {model} failed") + policies: Final = self._request.get( + "model_group_retry_policy", + self._core._resolved.model_group_retry_policy, # pyright: ignore[reportPrivateUsage] # the driver serves its own core + ) + raised: Final = policy_lookup_error(original, model, self._core._written_retry_policy, policies) # pyright: ignore[reportPrivateUsage] # see above + level: Final = event.get("level") + self._last_error = raised + self._level_errors[level if isinstance(level, int) else 0] = raised + + def _stamped_budget(self, event: Mapping[str, object]) -> Mapping[str, object]: + """A ``retries_opened`` event as ``_metadata["max_retries"]`` reads it: a request + ``num_retries`` that is no count is stamped as given.""" + given: Final = self._request.get("num_retries") + return {**event, "max_retries": given} if "uncounted_retries" in request_retry_fields(given) else event + + def _count_raised(self, event: Mapping[str, object]) -> None: + """A hop's first failure reached the retry loop's read of a request ``num_retries`` that + raises: the hop ends with that error, raised from the failure the event names.""" + model: Final = str(event.get("model")) + original: Final = self._cause_error(event) or RuntimeError(f"router-core: the hop serving {model} failed") + raised: Final = retry_count_error(original, self._request.get("num_retries")) + level: Final = event.get("level") + self._last_error = raised + self._level_errors[level if isinstance(level, int) else 0] = raised + + def _cause_error(self, event: Mapping[str, object]) -> BaseException | None: + """The error the ``cause`` of a ``retry_logged`` or ``retry_policy_raised`` event names.""" + match event.get("cause"): + case "level": + level: Final = event.get("level") + return self._level_errors.get(level if isinstance(level, int) else 0, self._last_error) + case "pick": + at_level: Final = event.get("level") + held: Final = self._level_picks.get(at_level if isinstance(at_level, int) else 0) + return held if held is not None else self._picked_error(event) + case _: + return self._last_error + + def _picked_error(self, event: Mapping[str, object]) -> BaseException: + """The error a pick an event names raised, as a failed step with that error raises it.""" + model: Final = str(event.get("failed_model") or event.get("model")) + return self._core._failure_error(event, model, None, request=self._request) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + + def _pick_raised(self, error: Exception, *, logged: bool) -> Mapping[str, object]: + """A pick raised on the host (a strategy, a tag filter, a pre-routing hook): the try fails with + that error, which the Rust walk retries and falls back from like any attempt's. ``logged`` + when the async pick's ``except`` reports it, as ``_async_get_available_deployment`` does.""" + if isinstance(error, RustCoreUnsupported): + raise error + if logged: + self._log_pick_failure(error) + else: + mark_deployment_selection_failure(error) + self._chain_to_hop_first(error, self._level) + self._last_error = error + self._level_errors[self._level] = error + return {**attempt_failure(error), "raised": True} + + def _log_pick_failure(self, error: BaseException) -> None: + """The ``except`` of ``_async_get_available_deployment``, which only the async selection runs: + the router's strategy may have no async selector, and a custom strategy replaces the method. + ``_aimage_generation`` logs a refused pick itself when the request set up no logging object.""" + core: Final = self._core + assigned: Final = "_core_async_get_available_deployment" in core.__dict__ + custom: Final = assigned or core._custom_strategy is not None # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if not custom and core._picks_async(): # pyright: ignore[reportPrivateUsage] # see above + mark_deployment_selection_failure(error) + _log_refused_pick(self._request, error) + if self._plan.kind == "image" and self._request.get("litellm_logging_obj") is None: + _log_refused_image_pick(self._request, error) + + def _pick_failed(self, event: Mapping[str, object]) -> None: + """A pick failed in the Rust core: its error is raised once, however many events name it.""" + error: Final = self._picked_error(event) + level: Final = event.get("level") + self._level_picks[level if isinstance(level, int) else 0] = error + if not self._sync: + self._log_pick_failure(error) + + def _unroutable_error(self, model: object) -> BaseException: + """What a hop to a dict target's non-string ``model`` raises: no group answers ``None``, and + any other value fails the level's fallback lookup, which hashes it and then searches it.""" + if model is None: + return self._core._failure_error({"error_kind": "unknown_model"}, "None", None) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + try: + _ = model in {} or "/" in model # pyright: ignore[reportOperatorIssue] # raising is the point + except TypeError as error: + return error + return TypeError(f"{model!r} is not a model group") + + def _level_failed(self, event: Mapping[str, object]) -> None: + """A level raises the error a pick raised (``level_failed``).""" + level: Final = event.get("level") + at_level: Final = level if isinstance(level, int) else 0 + picked: Final = ( + self._picked_error(event) + if event.get("handling") is True + else self._level_picks.pop(at_level, None) or self._picked_error(event) + ) + if event.get("handling") is True: + # Raised by the level's fallback handling, while the level's error was handled. + picked.__context__ = self._level_errors.get(at_level, self._last_error) + self._level_errors[at_level] = picked + if at_level == self._base_level: + self._explains_failure = False + return + stamp: Final = self._pending_stamps.pop(at_level, None) + if stamp is not None and type(picked) in self._core._host.dynamic().exception_types: # pyright: ignore[reportPrivateUsage] # the driver serves its own core + # The exhausted retry loop raised this pick's error: stamped as `retries_exhausted` stamps. + picked.max_retries, picked.num_retries = stamp # pyright: ignore[reportAttributeAccessIssue] # Python's router stamps the same attributes + self._chain_to_hop_first(picked, at_level) + self._level_errors[at_level] = picked + + def _carry_superseded_usage(self) -> None: + """``carry_superseded_attempt_usage``, before each same-group retry and each fallback hop.""" + from litellm.router_utils.fallback_event_handlers import carry_superseded_attempt_usage + + carry_superseded_attempt_usage(self._request) + + def _order_chain_failed(self, event: Mapping[str, object]) -> None: + """The order path of ``async_function_with_fallbacks_common_utils`` raised: the chain's last + hop's error, which the hop's level ended on, unless the level keeps its own.""" + level: Final = event.get("level") + at: Final = level if isinstance(level, int) else 0 + last_hop: Final = self._level_errors.get(at + 1) + if event.get("last_hop") is True and last_hop is not None: + self._level_errors[at] = last_hop + if at == self._base_level: + self._explains_failure = False + + def _logged_off_the_loop(self) -> bool: + """Whether litellm logged the attempt's failure on a thread with no running loop. An async + call's mocked failure raises in ``completion``, which ``acompletion`` runs in an executor + thread; a sync stream's first chunk logs in a ``threading.Thread``.""" + from litellm.utils import should_run_mock_completion + + if self._sync: + return self._reading_first_chunk + kwargs: Final[Mapping[str, object]] = self._latest_call[1].kwargs if self._latest_call is not None else {} + return self._plan.kind in ("chat", "text") and should_run_mock_completion( + kwargs.get("mock_response"), kwargs.get("mock_tool_calls"), kwargs.get("mock_timeout") + ) + + def _cooldown_set(self, event: Mapping[str, object]) -> None: + """An attempt's failure cooled its deployment down: ``set_cooldown_deployments`` as the failure + callback runs it. Outside a running loop it raises, which ``CustomLogger.log_event`` swallows; + a sync walk with a private loop has one (``cooldown_event_on_own_loop``).""" + deployment_id: Final = event.get("deployment_id") + seconds: Final = event.get("cooldown_time") + if self._logged_off_the_loop() or not isinstance(deployment_id, str): + return + fire: Final = self._core.cooldown_event_on_own_loop if self._plan.on_its_own_loop else self._core.cooldown_event + with contextlib.suppress(RuntimeError): + fire( + deployment_id, + getattr(self._last_error, "status_code", ""), + self._python_cooldown_time(seconds) if isinstance(seconds, float) else None, + ) + + def _python_cooldown_time(self, seconds: float) -> float: + """``_time_to_cooldown`` of ``deployment_callback_on_failure`` for the attempt that failed.""" + deployment: Final[Mapping[str, object]] = self._latest_call[0] if self._latest_call is not None else {} + configured: Final = first_cooldown( + as_str_mapping(deployment.get("model_info")), + self._request, + as_str_mapping(deployment.get("litellm_params")), + ) + return self._core.python_cooldown(seconds, configured, self._last_error) + + def _selected(self, event: Mapping[str, object]) -> None: + """``_acompletion``'s service event for the pick it awaited, and its ``_track_deployment_metrics``.""" + seconds: Final = event.get("pick_seconds") + if self._plan.kind == "chat" and not self._sync and isinstance(seconds, float): + end_time: Final = unix_time() + self._core.service_logged("async_get_available_deployment", end_time - seconds, end_time, self._request) + self._core.track_deployment_pick(event.get("deployment_id"), self._request) + + def _fallbacks_failed(self, event: Mapping[str, object]) -> None: + """The ``except`` around the regular fallbacks of ``async_function_with_fallbacks_common_utils``: + it logs what the ``try`` raised, the last hop's error or the level's own. A level with no + target raises its own error into it, except where no fallback list was set to search.""" + import litellm.exceptions as litellm_exceptions + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + + level: Final = event.get("level") + at: Final = level if isinstance(level, int) else 0 + own: Final = self._level_errors.get(at, self._last_error) + request: Final = self._request + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if event.get("chain") is not True: + policy_list: Final = request.get("content_policy_fallbacks", walked["content_policy_fallbacks"]) + if not isinstance(own, litellm_exceptions.ContentPolicyViolationError) or policy_list is None: + if request.get("fallbacks", walked["fallbacks"]) is None: + return + self._explain_no_fallback_group(own, at) + raised: Final = self._level_errors.get(at + 1, own) if event.get("last_hop") is True else own + self.outcomes.failed_fallbacks( + RoutingContext(self._core), # pyright: ignore[reportArgumentType] # the read uses registry() and health() + truncate_fallback_error_detail(redact_string(str(raised))), + get_parent_otel_span_from_kwargs(dict(request)), + ) + + def _explain_no_fallback_group(self, primary: BaseException | None, at: int) -> None: + """The primary level names the missing fallback group on its error before raising it into + the ``except``, so the log reads it and no outcome message follows.""" + if ( + at == self._base_level == 0 + and self._core._host.dynamic().expose_router_debug_in_errors # pyright: ignore[reportPrivateUsage] # the driver serves its own core + and primary is not None + and hasattr(primary, "message") + ): + primary.message += self._explanation(primary, str(self._request.get("model"))) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above # rebind-ok: the message is what this explains + self._explains_failure = False + + def take_events(self, events: object) -> None: # noqa: C901 # one branch per router-core event + for raw_event in _as_list(events): + event = as_str_mapping(raw_event) + if event is None: + continue + if event.get("kind") == "retry": + self._carry_superseded_usage() + continue + if event.get("kind") == "cooldown_set": + self._cooldown_set(event) + continue + if event.get("kind") == "selected": + self._selected(event) + continue + if event.get("kind") == "retries_opened": + self._walk.retries_opened(self._stamped_budget(event)) + continue + if event.get("kind") == "retry_budget": + self._walk.retry_budget(event) + continue + if event.get("kind") == "pick_failed": + self._pick_failed(event) + continue + if event.get("kind") == "order_chain_failed": + self._order_chain_failed(event) + continue + if event.get("kind") == "mock_failed": + resolved = self._core._resolved # pyright: ignore[reportPrivateUsage] # the driver serves its own core + mocked = mock_error( + str(event.get("mock")), + str(event.get("model")), + event.get("num_retries"), + self._request, + { + "fallbacks": resolved.fallbacks, + "context_window_fallbacks": resolved.context_window_fallbacks, + "content_policy_fallbacks": resolved.content_policy_fallbacks, + }, + ) + level = event.get("level") + self._last_error = mocked + self._level_errors[level if isinstance(level, int) else 0] = mocked + continue + if event.get("kind") == "retry_policy_raised": + self._policy_raised(event) + continue + if event.get("kind") == "retry_count_raised": + self._count_raised(event) + continue + if event.get("kind") == "level_failed": + self._level_failed(event) + continue + if event.get("kind") == "fallbacks_failed": + self._fallbacks_failed(event) + continue + if event.get("kind") == "fallback": + hop_to = event.get("level") + self._hop_first.pop(hop_to if isinstance(hop_to, int) else 0, None) + self._walk.hop(event) + self._carry_superseded_usage() + hop_level = event.get("level") + self._level = hop_level if isinstance(hop_level, int) else self._level + if "unroutable" in event: + self._level_errors[self._level] = self._unroutable_error(event["unroutable"]) + continue + if event.get("kind") == "blocked": + level = event.get("level") + blocked = self._blocked_error(event) + self._stamp_blocked(blocked) + self._last_error = blocked + self._level_errors[level if isinstance(level, int) else 0] = blocked + continue + if event.get("kind") == "falling_back": + level = event.get("level") + at = level if isinstance(level, int) else 0 + self._explains_failure = self._explains_failure or at == self._base_level + self._hint_level_failure(self._level_errors.get(at, self._last_error), str(event.get("model"))) + continue + if event.get("kind") == "fallback_done": + level = event.get("level") + at = level if isinstance(level, int) else 0 + self._level = at + self.outcomes.done( + event, + self._level_errors.get(at, self._last_error), + self._level_errors.get(at + 1), + self._walk.hop_kwargs(event), + ) + if event.get("succeeded") is not True: + self._core.trigger_fallback_cooldown( + self._request, + self._level_errors.get(at + 1), + self._plan.kind in _GENERIC_API_KINDS, + own_loop=self._plan.on_its_own_loop, + ) + continue + if event.get("kind") == "retry_logged": + logged = self._logged_error(event) + self._walk.log_retry(event, logged) + logged_level = event.get("level") + if event.get("attempted_retries") == 0 and logged is not None: + self._hop_first[logged_level if isinstance(logged_level, int) else 0] = logged + continue + if event.get("kind") != "retries_exhausted": + continue + if event.get("picked") is True: + # The level raises the last retry's pick error, which its `level_failed` names. + exhausted_level = event.get("level") + self._pending_stamps[exhausted_level if isinstance(exhausted_level, int) else 0] = ( + event.get("max"), + event.get("attempted"), + ) + continue + error: BaseException | None = self._last_error + # `async_function_with_retries` stamps the error its exhausted retry loop raises. + if error is not None and type(error) in self._core._host.dynamic().exception_types: # pyright: ignore[reportPrivateUsage] # the driver serves its own core + error.max_retries = event.get("max") # pyright: ignore[reportAttributeAccessIssue] # Python's router stamps the same attributes + error.num_retries = event.get("attempted") # pyright: ignore[reportAttributeAccessIssue] # same + + def _call(self, step: Mapping[str, object]) -> "tuple[Mapping[str, object], AttemptCall]": + name: Final = step.get("model") + requested: Final = str(name) if name is not None else str(self._request.get("model")) + deployment: Final = ( + # A default deployment whose id is not a string is called as Python copies it, id and all. + as_str_mapping(default_deployment_copy(self._core._resolved.default_deployment, requested)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if step.get("deployment_id") == DEFAULT_DEPLOYMENT_COPIED_ID + else for_requested_model(typed_deployment(as_str_mapping(step.get("deployment")) or {}), requested) + ) + self._attempts += 1 + self._attempt_started = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline's api_call_start_time + level: Final = step.get("fallback_level") + # The request params the walk's dict targets merged in (`kwargs.update(target)`). + params: Final = as_str_mapping(step.get("params")) + call: Final = attempt_call( + self._core._port, # pyright: ignore[reportPrivateUsage] # the driver serves its own core + self._plan, + self._request if params is None else {**self._request, **params}, + deployment, + group=requested, + requested=requested, + fallback_level=level if isinstance(level, int) else 0, + ) + self._strip_undecryptable_reasoning(step, deployment, call.kwargs) + shaped: Final = shape_attempt_kwargs(call.kwargs, deployment) + # The walk's metadata for the attempt's level (retries, fallback hops) and its hop kwargs. + call.kwargs = self._walk.stamp(step, shaped) + field: Final = self._plan.metadata_field + if isinstance(call.served, dict) and field in call.kwargs: + call.served[field] = call.kwargs[field] + self._served = (requested, call.served) + self._latest_call = (deployment, call) + if call.refused is None: + self._core._endpoints.counters.started(self._plan, call) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + return deployment, call + + def _strip_undecryptable_reasoning( + self, step: Mapping[str, object], deployment: Mapping[str, object], kwargs: Mapping[str, object] + ) -> None: + """``_update_kwargs_with_deployment`` on a fallback hop (``_is_fallback_hop``): the encrypted + reasoning the target cannot decrypt is dropped; reasoning with no origin marker came from + the deployment that just failed.""" + depth: Final = step.get("fallback_depth") + if not isinstance(depth, int) or depth <= 0: + return + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + RoutingContext(self._core), # pyright: ignore[reportArgumentType] # the check reads registry() + kwargs.get("input"), + kwargs.get("messages"), + (deployment,), + unmarked_origin=self._walk.deployment_that_just_failed(step) or self._cut_deployment, + ) + + @property + def response(self) -> object: + return self._last_response + + @property + def last_error(self) -> BaseException | None: + return self._last_error + + def _stream_attempt( + self, step: Mapping[str, object], kwargs: Mapping[str, object], slot: "contextlib.ExitStack | None" = None + ) -> StreamAttempt: + def number(key: str) -> int: + value: Final = step.get(key) + return value if isinstance(value, int) else 0 + + order: Final = step.get("target_order") + deployment_id: Final = step.get("deployment_id") + return StreamAttempt( + core=self._core, + op=self._op, + request=self._request, + kwargs=kwargs, + group=str(step.get("model") or step.get("model_group")), + order=order if isinstance(order, int) else None, + depth=number("fallback_depth"), + level=number("fallback_level"), + deployment_id=deployment_id if isinstance(deployment_id, str) else None, + attempted_retries=number("retry"), + slot=slot, + ) + + def _failed( + self, + step: Mapping[str, object], + deployment: Mapping[str, object], + call: "AttemptCall | None", + error: BaseException, + ) -> Mapping[str, object]: + # What litellm's failure logging read, before the attempt helper's `except` wrote on it. + detail: Final = str(error) + served: Final = call.served if call is not None else {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + stamp_error(self._plan, error, deployment, served) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + self._core._endpoints.attempt_failed(self._plan, error, deployment, call, str(step.get("model"))) # pyright: ignore[reportPrivateUsage] # see above + self._last_error = error + level: Final = step.get("fallback_level") + retry: Final = step.get("retry") + if isinstance(retry, int) and retry > 0: + self._chain_to_hop_first(error, level if isinstance(level, int) else 0) + self._level_errors[level if isinstance(level, int) else 0] = error + logged: Final = logged_call( + call.host_op if call is not None else self._plan.host_op, + served or self._request, # pyright: ignore[reportUnknownArgumentType] # see above + self._attempt_started, + ) + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + return { + **attempt_failure(error), + "detail": detail, + **self._core.failure_exits(error, logged, model_info.get("id")), + # `async_deployment_callback_on_failure` does not run for a sync call, and skips a batch retrieve. + "uncounted_request": self._plan.sync or is_batch_retrieve_call_type(logged.get("call_type")), + } + + def _chain_to_hop_first(self, error: BaseException, level: int) -> None: + """A retry's failure is raised while the hop's first failure is handled (the retry loop + runs in its ``except`` block), so it chains to it.""" + first: Final = self._hop_first.get(level) + if first is not None and first is not error: + error.__context__ = first # rebind-ok: chained as the retry loop's raise chains it + + def _succeeded( + self, call: "AttemptCall", response: object, host_op: str, stream: "StreamAttempt | None" = None + ) -> Mapping[str, object]: + self._last_response = response + tokens: Final = total_tokens(response) + # A wrapped stream reports its success or failure when it ends; the attempt counts the + # request and the tokens read so far, which the end's report leaves out. + # A realtime attempt counts its session's success itself (`attempt`), when it ended cleanly. + deferred: Final = (stream is not None and stream.reports_outcome) or reports_own_success(call) + if stream is not None and deferred: + stream.counted_tokens = tokens # rebind-ok: what the end's report leaves out + return { + "ok": True, + "total_tokens": tokens, + "blocked": self._blocked(response, host_op), + "deferred": deferred, + "tracks_usage": self._tracks_usage(call, response, deferred), + } + + def _tracks_usage(self, call: "AttemptCall", response: object, deferred: bool) -> bool: + """Whether the attempt counts against its deployment's usage (``_deployment_tracks_usage``). A + ``None`` response counts nothing (``increment_deployment_usage_for_response``), and an attempt + that reports no success then is a realtime session, which logs none to count it either.""" + if deferred and response is None: + return False + model: Final = call.kwargs.get("model") + return self._core.deployment_tracks_usage( + (as_str_mapping(call.kwargs.get("model_info")) or {}).get("id"), + model if isinstance(model, str) else call.group, + ) + + async def _refused( + self, + step: Mapping[str, object], + deployment: Mapping[str, object], + call: "AttemptCall", + error: BaseException, + ) -> Mapping[str, object]: + """A routing pre-call check refused the attempt (``async_routing_strategy_pre_call_checks``): + a rate limit cools the deployment down (``set_cooldown_deployments``); the attempt's failure + is recorded by nothing else.""" + import litellm.exceptions as litellm_exceptions + + core: Final = self._core + if isinstance(error, litellm_exceptions.RateLimitError): + deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id") + if isinstance(deployment_id, str): + failure: Final = {**attempt_failure(error), "caller_scoped": core.per_user_deployment(deployment_id)} + report: Final = core._handle.report_failure(deployment_id, failure, uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + done: Final = await core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # see above + _ = core.cooled_down(done, deployment_id, error.status_code) + return {**self._failed(step, deployment, call, error), "recorded": True} + + def _pick_kwargs(self, step: Mapping[str, object]) -> Mapping[str, object]: + """The request kwargs the pick sees: the request with its level's retry count.""" + count: Final = self._walk.request_retry_count(step) + if count is None: + return self._request + field: Final = self._plan.metadata_field + metadata: Final = as_str_mapping(self._request.get(field)) or {} + return {**self._request, field: {**metadata, "request_retry_count": count}} + + def _with_deployment(self, step: Mapping[str, object], resolved: object) -> Mapping[str, object]: + return step if resolved is step.get("deployment") else {**step, "deployment": resolved} + + async def _resolve_pods(self, step: Mapping[str, object]) -> Mapping[str, object]: + """``async_resolve_pods_after``: a picked deployment with Kubernetes pod discovery on is + sent to one of its service's pods.""" + if self._plan.kind in _UNPICKED_KINDS or step.get("deployment") is None: + return step + discovery: Final = self._core._pick_discovery(sync=False) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if discovery is None: + return step + resolved: Final[object] = await discovery.async_resolve_deployment( + step.get("deployment"), self._pick_kwargs(step) + ) + return self._with_deployment(step, resolved) + + def _resolve_pods_sync(self, step: Mapping[str, object]) -> Mapping[str, object]: + """``resolve_pods_after``, on the sync pick.""" + if self._plan.kind in _UNPICKED_KINDS or step.get("deployment") is None: + return step + discovery: Final = self._core._pick_discovery(sync=True) # pyright: ignore[reportPrivateUsage] # see _resolve_pods + if discovery is None: + return step + return self._with_deployment( + step, discovery.resolve_deployment(step.get("deployment"), self._pick_kwargs(step)) + ) + + def _default_copy(self, step: Mapping[str, object]) -> Mapping[str, object]: + """The default deployment as Python copied it for the call that fails on it: the error is + stamped from this copy, never from the id router-core serves it under.""" + copied: Final = default_deployment_copy(self._core._resolved.default_deployment, str(step.get("model"))) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + return as_str_mapping(copied) or {} + + def _default_call_error(self, step: Mapping[str, object]) -> BaseException | None: + """The error a call to a default deployment every call fails to raises (its first + reads of the copy, before any endpoint runs): nothing records it against a deployment.""" + if step.get("deployment_id") != DEFAULT_DEPLOYMENT_FAILING_ID: + return None + return default_deployment_call_error(self._core._resolved.default_deployment) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + + async def _passthrough_attempt(self, step: Mapping[str, object]) -> Mapping[str, object]: + """``_ageneric_api_call_with_fallbacks_helper`` with ``passthrough_on_no_deployment``: the pick + failed, so the handler is called under the attempt's name. Nothing was picked, so nothing is + stamped on a failure or recorded against a deployment.""" + passthrough: Final = self._passthrough + if passthrough is None: + raise RuntimeError("router-core asked for an unrouted attempt the host did not offer") + name: Final = str(step.get("model")) + params: Final = as_str_mapping(step.get("params")) + self._attempts += 1 + self._attempt_started = datetime.now() # noqa: DTZ005 # see _call + try: + answered: Final = await passthrough(self._request if params is None else {**self._request, **params}, name) + except Exception as error: # noqa: BLE001 # handed to the Rust call loop like any attempt failure + return {**self._failed(step, {}, None, error), "recorded": True} + self._last_response = answered + self._served = (name, self._request) + return {"ok": True, "total_tokens": total_tokens(answered)} + + async def attempt(self, step: Mapping[str, object]) -> Mapping[str, object]: + if self._passthrough is not None and not step.get("deployment_id"): + return await self._passthrough_attempt(step) + step = await self._resolve_pods(step) # rebind-ok: the picked deployment, resolved to a pod + deployment: Mapping[str, object] = as_str_mapping(step.get("deployment")) or {} # rebind-ok: the picked one + call: AttemptCall | None = None # rebind-ok: built inside the try + # Whether a failure raised now runs the failure callbacks: none raised before the litellm + # call or after it answered does, nor any but the first on a shared logging object. + logs_failure: bool = False # rebind-ok: set around the litellm call + reading_stream: StreamAttempt | None = None # rebind-ok: set while the first chunk is read + refused_call: Final = self._default_call_error(step) + if refused_call is not None: + return {**self._failed(step, self._default_copy(step), None, refused_call), "recorded": True} + try: + deployment, call = self._call(step) # rebind-ok: rebound as the Python router rebinds it + if call.refused is not None: + await log_refused_call(call, call.refused) + raise call.refused + host: Final = self._core._host # pyright: ignore[reportPrivateUsage] # the driver serves its own core + try: + compacted: Final = await compacted_attempt(self._core._port, self._plan, deployment, call) # pyright: ignore[reportPrivateUsage] # see above + except Exception as refused: # noqa: BLE001 # raised before the provider call, so no cooldown reads it + return {**self._failed(step, deployment, call, refused), "recorded": True} + # The deployment's max_parallel_requests slot; an open chat stream takes it along. + port = self._core._port # pyright: ignore[reportPrivateUsage] # the driver serves its own core # rebind-ok: the same port on every attempt + with enter_deployment_slot(port, self._plan, deployment, call.kwargs) as slot: + # A search tool is no deployment: nothing to check; Python's `_arerank` and + # `amoderation` run no routing pre-call checks either. + if self._plan.kind not in _NO_ROUTING_PRE_CALL_CHECKS: + try: + await routing_pre_call_checks( + self._core._checks.routing_callbacks(), # pyright: ignore[reportPrivateUsage] # see above + as_str_mapping(step.get("deployment")) or {}, + ) + except Exception as refused: # noqa: BLE001 # handed to the Rust call loop as the attempt's failure + # Only `_acompletion` hands the check its logging object. + if self._plan.kind == "chat": + _log_refused_pick(self._request, refused) + return await self._refused(step, deployment, call, refused) + first_to_log: Final = not failure_already_logged(call.kwargs, "async_failure") + logs_failure = first_to_log # rebind-ok: whether a failure here is this attempt's to log + pending: Final[object] = ( + self._core._endpoints.create_file_everywhere( # pyright: ignore[reportPrivateUsage] # see above + _with_target_order(call.kwargs, step.get("target_order")), + call.model_name, + self._request.get("specific_deployment"), + ) + if self._plan.kind == "create_file" + else host.invoke(call.host_op)(**compacted) + ) + # The generic helper awaits whatever the handler returns, as the Python router does. + awaited: Final = self._plan.kind == "generic" or inspect.isawaitable(pending) + answered: Final[object] = await pending if awaited else pending # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType] # see above + logs_failure = False # rebind-ok: see above + self._core._endpoints.check_response(self._plan, answered, call) # pyright: ignore[reportPrivateUsage, reportUnknownArgumentType] # see above + stream: Final = self._stream_attempt(step, call.kwargs, slot) + logs_failure = first_to_log # rebind-ok: the stream's own failure logging, for its first chunk + reading_stream = stream # rebind-ok: None until the attempt opens a stream + response: Final = await open_attempt_stream(stream, answered) # pyright: ignore[reportUnknownArgumentType] # see above + logs_failure = False # rebind-ok: see above + # `self.success_calls[model_name] += 1`, inside the try: a counter replaced with a + # plain dict fails the attempt. + self._core._endpoints.counters.succeeded(self._plan, call) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # every attempt failure is classified and handed to the Rust call loop + failed: Final = self._failed(step, deployment, call, error) + if reading_stream is None or not logs_failure: + return {**failed, "recorded": not logs_failure} + # `_dispatch_stream_failure`: the stream logs its failure in a task of its own. + reading_stream.report_first_chunk_failure(error) + return {**failed, "recorded": True} + served: Final = self._succeeded(call, response, call.host_op, stream) + # The session re-pin the Python core's success callback runs; a wrapped stream runs it when it ends. + if not served.get("deferred") and self._core.outcome_callbacks(call.kwargs, sync=False).get("success", True): + await self._core.repin_after_success(stream.logged(), response) + session_deployment: Final = clean_realtime_session_deployment(call) + if session_deployment is not None: + await self._core.count_realtime_session_success(session_deployment) + return served + + def attempt_sync(self, step: Mapping[str, object]) -> Mapping[str, object]: + step = self._resolve_pods_sync(step) # rebind-ok: see attempt + deployment: Mapping[str, object] = as_str_mapping(step.get("deployment")) or {} # rebind-ok: see attempt + call: AttemptCall | None = None # rebind-ok: see attempt + logs_failure: bool = False # rebind-ok: see attempt + self._reading_first_chunk = False + refused_call: Final = self._default_call_error(step) + if refused_call is not None: + return {**self._failed(step, self._default_copy(step), None, refused_call), "recorded": True} + try: + deployment, call = self._call(step) # rebind-ok: rebound as the Python router rebinds it + if call.refused is not None: + log_refused_call_sync(call, call.refused) + raise call.refused + try: + self._core._endpoints.sync_pre_call_checks(self._plan, deployment, self._request) # pyright: ignore[reportPrivateUsage] # see above + except Exception as refused: # noqa: BLE001 # handed to the Rust call loop like any attempt failure + # The sync path records no outcome for a refused pre-call check: no cooldown. + return {**self._failed(step, deployment, call, refused), "recorded": True} + first_to_log: Final = not failure_already_logged(call.kwargs, "sync_failure") + logs_failure = first_to_log # rebind-ok: see attempt + answered: Final = self._core._host.invoke(call.host_op)(**call.kwargs) # pyright: ignore[reportPrivateUsage] # a sync endpoint + logs_failure = False # rebind-ok: see attempt + if inspect.isawaitable(answered): + raise _NotSynchronous(f"the host's {call.host_op} endpoint is not synchronous") + self._core._endpoints.check_response(self._plan, answered, call) # pyright: ignore[reportPrivateUsage] # see above + stream: Final = self._stream_attempt(step, call.kwargs) + logs_failure = first_to_log # rebind-ok: see attempt + self._reading_first_chunk = True + response: Final = open_attempt_stream_sync(stream, answered) + self._reading_first_chunk = False + logs_failure = False # rebind-ok: see attempt + self._core._endpoints.counters.succeeded(self._plan, call) # pyright: ignore[reportPrivateUsage] # see attempt + except _NotSynchronous: + raise + except Exception as error: # noqa: BLE001 # see attempt + return {**self._failed(step, deployment, call, error), "recorded": not logs_failure} + return self._succeeded(call, response, "", stream) + + async def select(self, step: Mapping[str, object]) -> object: + request: Final = self._request + messages, input = self._plan.pick_input(request) + if step.get("stage") == "callback_filters": + answer, error = await self._core._answer_callback_filters(step, request, messages) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if isinstance(error, Exception): + # The try failed with the filter's error: the level raises it if nothing else serves. + _ = self._pick_raised(error, logged=True) + return answer + try: + return await self._core._answer_select(step, request, messages, input) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + except Exception as error: # noqa: BLE001 # handed to the Rust call loop as the try's failure + return self._pick_raised(error, logged=True) + + def select_sync(self, step: Mapping[str, object]) -> object: + request: Final = self._request + try: + return self._core._answer_select_sync(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # see select + return self._pick_raised(error, logged=False) + + async def pre_route(self, step: Mapping[str, object]) -> object: + request: Final = self._request + try: + return await self._core._pre_route(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # see select + return self._pick_raised(error, logged=True) + + def pre_route_sync(self, step: Mapping[str, object]) -> object: + request: Final = self._request + try: + return self._core._custom_pick_sync(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # see select + return self._pick_raised(error, logged=False) + + async def allow_fallback(self, target: str) -> object: + """`_is_fallback_target_authorized`, then `_is_fallback_target_within_budget`. A check that raises + ends the chain outside any hop's ``try``, as a chain's own failure: the level raises what it + would have (``async_function_with_fallbacks_common_utils`` catches it), and its note or an + order path reads the error as the next level's.""" + from litellm.router_utils.fallback_event_handlers import get_pre_routing_selection + from litellm.types.router import FallbackSkipReason + + ctx: Final = RoutingContext(self._core) # pyright: ignore[reportArgumentType] # the checks read registry() and settings() + request: Final = dict(self._request) + try: + if target != get_pre_routing_selection(request) and not await ctx.fallback_access_allowed( + model=target, request_kwargs=request + ): + self._walk.targets().record_skip(target, FallbackSkipReason.NOT_AUTHORIZED) + return False + if not await ctx.fallback_budget_allowed(model=target, request_kwargs=request): + self._walk.targets().record_skip(target, FallbackSkipReason.OVER_BUDGET) + return False + except Exception as error: # noqa: BLE001 # handed to the Rust call loop, which ends the chain + self._level_errors[self._level + 1] = error + return {**attempt_failure(error), "raised": True} + return True + + def refuse_fallbacks(self, step: Mapping[str, object]) -> bool: + """``run_async_fallback``'s check of a chain's dict targets: a refused chain ends the walk + with the check's error, which the first hop's level would have raised.""" + refused: Final = rejected_fallback_target(_as_list(step.get("targets"))) + if refused is None: + return False + level: Final = step.get("level") + self._level_errors[(level if isinstance(level, int) else 0) + 1] = refused + return True + + def _head_response(self) -> object | None: + """``set_response_headers`` up to its usage read: the deployment id, the model group the + response was served as and the routers' decisions.""" + response: Final = prepare_response_for_header_attachment(self._last_response) + if response is None: + return None + name, kwargs = self._served + additional: Final = ensure_response_additional_headers(response) + apply_response_model_id(response, find_deployment_metadata(kwargs)) + additional["x-litellm-model-group"] = name + apply_quality_router_decision_headers(additional, kwargs) + additional.update(complexity_router_decision_headers(kwargs)) + return response + + def _count_response(self, response: object, step: Mapping[str, object]) -> object: + """What the wrappers above `make_call` stamp: ``async_function_with_retries``'s retry count, + then the fallback count of ``async_function_with_fallbacks`` and ``run_async_fallback``.""" + retries: Final = step.get("attempted_retries") + if isinstance(retries, int): + max_retries: Final = step.get("max_retries") + response = add_retry_headers_to_response( # rebind-ok: gains the counts + response=response, + attempted_retries=retries, + max_retries=max_retries if isinstance(max_retries, int) and retries > 0 else None, + ) + fallbacks: Final = step.get("attempted_fallbacks") + attempted: Final = fallbacks if isinstance(fallbacks, int) else 0 + return add_fallback_headers_to_response( + response=self.outcomes.add_headers(response, attempted), attempted_fallbacks=attempted + ) + + def log_outcomes_sync(self) -> None: + if pending := self.outcomes.take_pending(): + from litellm.litellm_core_utils import asyncify + + asyncify.run_async_function(await_all, pending) # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + + def finish(self, step: Mapping[str, object]) -> object: + """The response with the router's headers, in the order ``make_call`` and the wrappers above it + stamp them: a usage read that raises leaves the counts off.""" + if not self._plan.sets_headers: + self.log_outcomes_sync() + return self._last_response + response: Final = self._head_response() + if response is None: + self.log_outcomes_sync() + return self._last_response + name: Final = self._served[0] + if name is not None: + core: Final = self._core + remaining: Final = as_str_mapping( + core.run_routing_cache_sync(lambda: core.health().usage(name, kind="remaining")) + ) + if remaining is not None: + apply_remaining_usage_headers(ensure_response_additional_headers(response), dict(remaining)) + counted: Final = self._count_response(response, step) + self.log_outcomes_sync() + return counted + + async def finish_async(self, step: Mapping[str, object]) -> object: + """``finish``, with the usage read on the call's own loop.""" + if not self._plan.sets_headers: + await await_all(self.outcomes.take_pending()) + return self._last_response + response: Final = self._head_response() + if response is None: + await await_all(self.outcomes.take_pending()) + return self._last_response + name: Final = self._served[0] + if name is not None: + remaining: Final = await self._core.health().usage(name, kind="remaining") + if isinstance(remaining, Mapping): + apply_remaining_usage_headers(ensure_response_additional_headers(response), dict(remaining)) + counted: Final = self._count_response(response, step) + await await_all(self.outcomes.take_pending()) + return counted + + def failed(self, step: Mapping[str, object]) -> BaseException: + model: Final = str(self._request.get("model")) + primary: Final = self._level_errors.get(self._base_level) + if primary is None: + return self._core._failure_error(step, model, self._last_error, request=self._request) # pyright: ignore[reportPrivateUsage] # see above + if ( + self._base_level == 0 + and self._explains_failure + and self._core._host.dynamic().expose_router_debug_in_errors # pyright: ignore[reportPrivateUsage] # see above + and hasattr(primary, "message") + ): + primary.message += self._explanation(primary, model) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + return primary + + def _hint_level_failure(self, error: BaseException | None, model: str) -> None: + """A level that reaches its fallback handling on a context-window or content-policy error + with no list of its own appends the hint to set one, before any hop runs, as every level of + the Python walk does: a stream's failure event, still queued then, reads the hint.""" + from litellm.router_utils.fallback_event_handlers import check_non_standard_fallback_format + + if ( + error is None + or not hasattr(error, "message") + or not self._core._host.dynamic().expose_router_debug_in_errors # pyright: ignore[reportPrivateUsage] # the driver serves its own core + ): + return + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # see above + fallbacks: Final = self._request.get("fallbacks", walked["fallbacks"]) + if check_non_standard_fallback_format(fallbacks=_as_list(fallbacks)): + return + hint: Final = self._kind_hint(error, model, fallbacks) + if hint: + error.message += hint # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above # rebind-ok: the router appends to the error it raises + + def _kind_hint(self, error: BaseException, model: str, fallbacks: object) -> str: + """The hint a context-window or content-policy error gets when its own list is unset.""" + import litellm.exceptions as litellm_exceptions + + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if isinstance(error, litellm_exceptions.ContextWindowExceededError): + if self._request.get("context_window_fallbacks", walked["context_window_fallbacks"]) is not None: + return "" + return ( + f"\nmodel={model}. context_window_fallbacks=None. fallbacks={fallbacks}.\n\n" + "Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" + ) + if isinstance(error, litellm_exceptions.ContentPolicyViolationError): + if self._request.get("content_policy_fallbacks", walked["content_policy_fallbacks"]) is not None: + return "" + return "\n" + format_content_policy_fallback_hint(model, None, fallbacks) # pyright: ignore[reportArgumentType] # the list as configured + return "" + + def _explanation(self, primary: BaseException, model: str) -> str: + """What ``async_function_with_fallbacks_common_utils`` appends to the error it re-raises + under ``expose_router_debug_in_errors``, from the resolved settings and what the call did.""" + import litellm.exceptions as litellm_exceptions + from litellm.router_utils.fallback_event_handlers import ( + check_non_standard_fallback_format, + fallback_lookup_groups, + get_fallback_model_group_for_lookup_groups, + ) + + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # the driver serves its own core + request: Final = self._request + # The request's own lists replace the router's (`kwargs.get("fallbacks", router)`). + fallbacks: Final = request.get("fallbacks", walked["fallbacks"]) + window_list: Final = request.get("context_window_fallbacks", walked["context_window_fallbacks"]) + policy_list: Final = request.get("content_policy_fallbacks", walked["content_policy_fallbacks"]) + lookup: Final = fallback_lookup_groups(request, model) + + def group_of(source: object) -> tuple[Sequence[object] | None, int | None]: + return get_fallback_model_group_for_lookup_groups(_as_list(source), lookup) # pyright: ignore[reportArgumentType] # the fallbacks config shape + + def outcome(group: Sequence[object] | None) -> str: + return format_fallback_outcome_message( + model, group, self._fallback_detail(primary), skipped=self._walk.targets().skipped + ) + + # `context_window_fallback_model_group`: only a chain the group matched takes precedence. + window_group: Final = ( + group_of(window_list)[0] + if isinstance(primary, litellm_exceptions.ContextWindowExceededError) and window_list is not None + else None + ) + if window_group is None and check_non_standard_fallback_format(fallbacks=_as_list(fallbacks)): + return outcome(_as_list(fallbacks)) + if window_group is not None: + return outcome(window_group) + # A content-policy list decides on its own: one with no match raises into the `except`. + if isinstance(primary, litellm_exceptions.ContentPolicyViolationError) and policy_list is not None: + return outcome(group_of(policy_list)[0]) + if fallbacks is None or not lookup: + return outcome(None) + found, generic = group_of(fallbacks) + group: Final[Sequence[object] | None] = ( + found if found is not None or generic is None else _as_list(_as_list(fallbacks)[generic]["*"]) # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType, reportUnknownArgumentType] # the entry the lookup matched + ) + return format_no_fallback_group_message(lookup, _as_list(fallbacks)) if group is None else outcome(group) # pyright: ignore[reportArgumentType] # the fallbacks config shape + + def _fallback_detail(self, primary: BaseException) -> str: + """``run_async_fallback`` raises its last hop's error, or the primary's when no hop ran.""" + return truncate_fallback_error_detail(redact_string(str(self._level_errors.get(1, primary)))) + + +__all__ = [ + "Resolved", + "RustCore", + "RustCoreUnsupported", + "RustHealth", + "RustRegistry", + "RustSettings", + "attempt_failure", + "failure_kind", + "total_tokens", + "unsupported_arguments", +] diff --git a/litellm/router_rust/invoke/README.md b/litellm/router_rust/invoke/README.md new file mode 100644 index 00000000000..d8562f99359 --- /dev/null +++ b/litellm/router_rust/invoke/README.md @@ -0,0 +1,13 @@ +# invoke + +The host side of `Host::invoke`: running one attempt against the one deployment router-core picked. Nothing here selects, retries, falls back or records cooldowns. + +- `shaping`: the kwargs an attempt sends to its deployment +- `endpoints`: what each public router method does before, around and after the routed call +- `streaming`: stream wrappers, so a stream that fails before its first token fails the attempt +- `slots`: a deployment's `max_parallel_requests` +- `walk`: the retry and fallback metadata stamped on each attempt +- `prompt_management`: prompt-management groups +- `websocket`: admission for each turn of a Responses websocket + +A Rust host replaces this with native endpoint calls. diff --git a/litellm/router_rust/invoke/__init__.py b/litellm/router_rust/invoke/__init__.py new file mode 100644 index 00000000000..faa2c13198d --- /dev/null +++ b/litellm/router_rust/invoke/__init__.py @@ -0,0 +1 @@ +"""The host side of ``Host::invoke``: one router attempt against one deployment.""" diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py new file mode 100644 index 00000000000..c1dc5162ccf --- /dev/null +++ b/litellm/router_rust/invoke/endpoints.py @@ -0,0 +1,2454 @@ +""" +The Rust router core's endpoints: what each public router method does before, around and after +the routed call, as the Python router's endpoint methods do it. + +An endpoint is either routed (the Rust core picks the deployment, retries and falls back, and the +host runs each attempt) or served directly (the Python router calls litellm's function without +routing: no model group, an id that names no deployment, a fan-out over every deployment). Every +call out goes through the host: ``host.invoke(op)``, where an op is a litellm function name or a +dotted path to one (``vector_stores.main.asearch``), looked up when it is called. +""" + +import asyncio +import copy +import dataclasses +import threading +import time +import uuid +from collections import defaultdict +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + add_fallback_headers_to_response, + add_retry_headers_to_response, +) +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name +from litellm.litellm_core_utils.core_helpers import without +from litellm.litellm_core_utils.routing_common_utils import without_router_only_kwargs +from litellm.responses.retrieve_stream_query import without_retrieve_stream_defaults +from litellm.router_api import RoutingContext +from litellm.router_rust.invoke.shaping import ( + ROUTER_CLIENTS_TARGET, + STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + Shaping, + as_dict, + deployment_params_with_request_reasoning_override, + drop_unsupported_classifier_reasoning_effort, + metadata_bucket, + stamp_deployment_num_retries, + stamp_failed_deployment_id, + stamp_retry_skip_deployment_id, + update_kwargs_with_deployment, +) +from litellm.router_strategy.complexity_router.context_compaction import ( + Surface, + compact_to_fit, + initialize_compaction_state, + surface_for_call, +) +from litellm.router_utils.endpoint_eligibility import endpoint_for_call, endpoint_selection + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.router_api import RouterRegistry + from litellm.router_rust.core import RustCore + from litellm.router_rust.steps.selection import StrategySelectors + from litellm.rust_bridge._native import RouterCoreHandle + from litellm.types.router import Deployment + + +class EndpointCore(Protocol): + """What the endpoints use of the core they serve: its host, Rust handle, settings, routing + cache, registry lookups and call drivers. ``RustCore`` provides it through ``_EndpointPort``; + a Rust host's endpoint layer would have to provide the same list.""" + + @property + def router(self) -> "RustCore": + """The core itself, for the provider handlers and helpers that take a router.""" + ... + + @property + def host(self) -> "PythonHost": ... + + @property + def handle(self) -> "RouterCoreHandle": ... + + @property + def resolved(self) -> "Resolved": ... + + @property + def shaping(self) -> Shaping: ... + + @property + def routing_cache(self) -> "DualCache": ... + + @property + def selectors(self) -> "StrategySelectors": ... + + @property + def endpoints(self) -> "RustEndpoints": ... + + async def drive( + self, + plan: "Plan", + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + passthrough: Callable[[Mapping[str, object], str], Awaitable[object]] | None = None, + prepared_already: bool = False, + ) -> object: ... + + def call_sync( + self, plan: "Plan", request: Mapping[str, object], overrides: Mapping[str, object] | None = None + ) -> object: ... + + def routing_callbacks(self) -> Iterable[object]: ... + + def resolve_alias(self, model: str) -> str | None: ... + + def group_deployments(self, model_name: str) -> Sequence[Mapping[str, object]] | None: + """``_core_get_model_list(model_name=...)`` on the host registry.""" + ... + + def routed_name(self, model: object) -> str: + """``RustCore.routed_name``: what a ``model`` that is no string raises, or the name it routes under.""" + ... + + def service_logged( + self, call_type: str, start_time: float, end_time: float, request: Mapping[str, object] + ) -> None: ... + + async def set_response_headers( + self, response: object, model_group: str | None, request_kwargs: Mapping[str, object] + ) -> object: ... + + def configured_deployment(self, model_id: str) -> "Deployment | None": + """``_core_get_deployment``: the deployment as the host registry holds it now.""" + ... + + def registry(self) -> "RouterRegistry": ... + + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the RouterCore signature + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: ... + + async def acompletion(self, *args: object, **kwargs: object) -> object: ... # kwargs-ok: endpoint payload + + +Kind = Literal[ # rebind-ok: rebound as the Python router rebinds it + "chat", + "text", + "embedding", + "image", + "transcription", + "speech", + "rerank", + "adapter", + "generic", + "generic_sync", + "batch", + "cancel_batch", + "vector_store_create", + "moderation", + "create_file", + "search", +] +Stamp = Literal["deployment", "retry_skip", "none"] # rebind-ok: rebound as the Python router rebinds it +CountKey = Literal["deployment", "group", "empty"] # rebind-ok: rebound as the Python router rebinds it + +# The text of the user message each attempt helper hands its pick in place of the request's own +# (`_aimage_generation`, `_aadapter_completion`, `_acreate_batch`, ...). +_PICK_TEXTS: Final[Mapping[str, str]] = MappingProxyType( + { + "image": "prompt", + "transcription": "prompt", + "speech": "prompt", + "adapter": "default text", + "batch": "files-api-fake-text", + "create_file": "files-api-fake-text", + "cancel_batch": "batch-api-fake-text", + "vector_store_create": "vector-store-api-fake-text", + } +) + + +def _user_message(content: object) -> list[dict[str, object]]: # mutable-ok: a fresh list, as each pick builds it + return [{"role": "user", "content": content}] + + +# Request keys the router reads to route the call; the deployment call never receives them. +ROUTING_KEYS: Final = frozenset( + { + "model", + "num_retries", + "disable_fallbacks", + "specific_deployment", + "_router_weights", + "_retry_skipped_deployment_ids", + "_excluded_deployment_ids", + "_target_order", + "original_function", + # The walk's own settings, which `async_function_with_fallbacks` pops before the attempt. + "model_group_retry_policy", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + } +) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Plan: + """How one endpoint runs its routed attempts.""" + + endpoint: str + host_op: str + kind: Kind + sync: bool = False + # The bucket `_update_kwargs_before_fallbacks` writes the requested group into; None for the + # endpoints that write `metadata.model_group` alone (no trace id, no alias). + prepare_field: str | None = "metadata" + function_name: str | None = None + splits_stream_timeout: bool = False + stamp: Stamp = "retry_skip" + total_key: CountKey | None = "deployment" + success_key: CountKey | None = "deployment" + fail_key: CountKey | None = "deployment" + # One attempt, no retries and no fallbacks. + single_attempt: bool = False + # The named parameter the endpoint takes beside **kwargs, sent between the deployment params and + # the request (`{**data, name: value, **kwargs}`). + named: tuple[str, ...] = () + + @property + def on_its_own_loop(self) -> bool: + """``function_with_fallbacks`` runs these sync endpoints' whole walk on a private event loop.""" + return self.sync and self.kind in ("chat", "embedding", "image") + + def pick_input(self, request: Mapping[str, object]) -> tuple[object, object]: + """The ``messages`` and ``input`` the endpoint's attempt helper hands its pick, which every + selection stage reads: the pre-call checks run when either is set.""" + match self.kind: + case "chat": + return request.get("messages"), None + case "text": + return _user_message(request.get("prompt")), None + case "embedding": + return None, request.get("input") + case "rerank" | "moderation" | "search": + return None, None + case "generic" | "generic_sync": + return request.get("messages"), request.get("input") + case _: + return _user_message(_PICK_TEXTS[self.kind]), None + + @property + def sets_headers(self) -> bool: + """Whether the response gets the router's headers (the sync generic wrapper adds none).""" + return self.kind != "generic_sync" + + @property + def stamps_failed_deployment(self) -> bool: + """Whether the attempt stamps ``failed_deployment_id`` on its error, which weighted failover + reads: every endpoint but file and batch creation (whose result belongs to the deployment + that served it), moderation's pass-through and search.""" + return self.kind not in ("batch", "cancel_batch", "create_file", "moderation", "search") + + @property + def metadata_field(self) -> str: + """Where `_update_kwargs_with_deployment` writes the deployment it routed to.""" + return get_router_metadata_variable_name(function_name=self.function_name) + + +def _generic(endpoint: str, host_op: str | None = None, *, sync: bool = False) -> Plan: + if sync: + return Plan( + endpoint=endpoint, + host_op=host_op or endpoint, + kind="generic_sync", + sync=True, + prepare_field="litellm_metadata", + function_name="generic_api_call", + splits_stream_timeout=_handler_name(host_op or endpoint) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + stamp="none", + fail_key="group", + single_attempt=True, + ) + return Plan( + endpoint=endpoint, + host_op=host_op or endpoint, + kind="generic", + prepare_field="litellm_metadata", + function_name="_ageneric_api_call_with_fallbacks", + splits_stream_timeout=_handler_name(host_op or endpoint) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + stamp="deployment", + fail_key="group", + ) + + +def _handler_name(host_op: str) -> str: + return host_op.rsplit(".", 1)[-1] + + +def invoked_handler_name(core: EndpointCore, plan: Plan) -> str | None: + """The generic handler's ``__name__`` (`original_generic_function.__name__`): the function the + host serves for the op, which decides the stream-timeout split and the responses socket.""" + function: Final[object] = core.host.invoke(plan.host_op) # pyright: ignore[reportUnknownMemberType] # the core's host + name: Final[object] = getattr(function, "__name__", None) + return name if isinstance(name, str) else None + + +# The endpoints whose attempt helper runs inside `make_call`'s `endpoint_selection`, under the +# name of its `original_function`. +_SELECTION_CALLS: Final[Mapping[str, str]] = MappingProxyType( + { + "acompletion": "_acompletion", + "completion": "_completion", + "atext_completion": "_atext_completion", + } +) + + +def selection_endpoint(core: EndpointCore, plan: Plan) -> str | None: + """The endpoint ``endpoint_selection`` names around the plan's pick. A sync generic call never + reaches ``make_call``, and a generic attempt reads the handler's name and module.""" + call: Final = _SELECTION_CALLS.get(plan.endpoint) + if call is not None: + return endpoint_for_call(call) + if plan.kind != "generic": + return None + function: Final[object] = core.host.invoke(plan.host_op) # pyright: ignore[reportUnknownMemberType] # the core's host + return endpoint_for_call(getattr(function, "__name__", None), getattr(function, "__module__", None)) + + +PLANS: Final[Mapping[str, Plan]] = MappingProxyType( + { + "acompletion": Plan( + "acompletion", "acompletion", "chat", splits_stream_timeout=True, stamp="deployment", named=("messages",) + ), + "completion": Plan( + "completion", + "completion", + "chat", + sync=True, + splits_stream_timeout=True, + stamp="deployment", + total_key=None, + success_key=None, + fail_key=None, + named=("messages",), + ), + "atext_completion": Plan( + "atext_completion", + "atext_completion", + "text", + splits_stream_timeout=True, + fail_key="group", + named=("prompt",), + ), + # `_asearch_with_fallbacks`: the search tool name routes as the model, with no deployment + # picked; its helper picks among the router's search tools. + "asearch": Plan( + "asearch", + "search.asearch", + "search", + prepare_field="litellm_metadata", + stamp="none", + total_key=None, + success_key=None, + fail_key=None, + ), + "aembedding": Plan("aembedding", "aembedding", "embedding", named=("input",)), + "embedding": Plan("embedding", "embedding", "embedding", sync=True, named=("input",)), + "aimage_generation": Plan("aimage_generation", "aimage_generation", "image", named=("prompt",)), + "image_generation": Plan( + "image_generation", + "image_generation", + "image", + sync=True, + prepare_field=None, + total_key="empty", + success_key="empty", + fail_key="empty", + named=("prompt",), + ), + "atranscription": Plan( + "atranscription", + "atranscription", + "transcription", + total_key="group", + success_key="group", + fail_key="group", + named=("file",), + ), + "aspeech": Plan( + "aspeech", + "aspeech", + "speech", + total_key="group", + success_key="group", + fail_key="group", + named=("input",), + ), + "arerank": Plan("arerank", "arerank", "rerank"), + "aadapter_completion": Plan( + "aadapter_completion", + "aadapter_completion", + "adapter", + prepare_field=None, + splits_stream_timeout=True, + fail_key="group", + ), + "acreate_batch": Plan( + "acreate_batch", + "acreate_batch", + "batch", + prepare_field="litellm_metadata", + function_name="_acreate_batch", + fail_key="group", + ), + "acancel_batch": Plan( + "acancel_batch", + "acancel_batch", + "cancel_batch", + prepare_field="litellm_metadata", + function_name="_acancel_batch", + fail_key="group", + ), + "acreate_file": Plan( + "acreate_file", + "acreate_file", + "create_file", + function_name="acreate_file", + stamp="none", + total_key=None, + success_key=None, + fail_key="group", + ), + "avector_store_create": Plan( + "avector_store_create", + "vector_stores.acreate", + "vector_store_create", + prepare_field=None, + function_name="avector_store_create", + stamp="none", + fail_key="group", + single_attempt=True, + ), + "amoderation": Plan( + "amoderation", + "amoderation", + "moderation", + stamp="none", + total_key=None, + success_key=None, + fail_key=None, + single_attempt=True, + ), + # The Python router's anthropic messages host op is `anthropic_messages`. + "aanthropic_messages": _generic("aanthropic_messages", "anthropic_messages"), + "anthropic_messages": _generic("anthropic_messages", "anthropic_messages"), + "aresponses": _generic("aresponses"), + "responses": _generic("responses", sync=True), + } +) + +# --- the generic endpoints ------------------------------------------------------------------ + +# endpoint -> host op, for the async generic endpoints routed through +# `_ageneric_api_call_with_fallbacks` when they carry a model. +_HOST_GENERIC: Final[Mapping[str, str]] = MappingProxyType( + { + "afile_delete": "afile_delete", + "afile_content": "afile_content", + "afile_list": "afile_list", + "aget_responses": "aget_responses", + "acancel_responses": "acancel_responses", + "acompact_responses": "acompact_responses", + "adelete_responses": "adelete_responses", + "alist_input_items": "alist_input_items", + "_arealtime": "_arealtime", + "acreate_realtime_client_secret": "acreate_realtime_client_secret", + "arealtime_calls": "arealtime_calls", + "acreate_realtime_transcription_session": "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret": "acreate_realtime_translation_client_secret", + "arealtime_translation_calls": "arealtime_translation_calls", + "_aresponses_websocket": "_aresponses_websocket", + "acreate_fine_tuning_job": "acreate_fine_tuning_job", + "acancel_fine_tuning_job": "acancel_fine_tuning_job", + "alist_fine_tuning_jobs": "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job": "aretrieve_fine_tuning_job", + "aimage_edit": "aimage_edit", + "allm_passthrough_route": "allm_passthrough_route", + "acreate_skill": "acreate_skill", + "alist_skills": "alist_skills", + "aget_skill": "aget_skill", + "adelete_skill": "adelete_skill", + "agenerate_content": "google_genai.agenerate_content", + "agenerate_content_stream": "google_genai.agenerate_content_stream", + "aocr": "ocr.aocr", + "adecisions": "decisions.adecisions", + "asystemone": "systemone.asystemone", + "aextraction": "extraction.aextraction", + **{f"avideo_{name}": f"videos.avideo_{name}" for name in ("generation", "list", "status", "content", "remix")}, + **{ + name: f"containers.{name}" + for name in ("acreate_container", "alist_containers", "aretrieve_container", "adelete_container") + }, + **{ + name: f"containers.endpoint_factory._generated_endpoints.{name}" + for name in ( + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + ) + }, + **{ + f"avector_store_{name}": f"vector_stores.main.a{name}" + for name in ("search", "retrieve", "list", "update", "delete") + }, + **{ + f"avector_store_file_{name}": f"vector_store_files.main.a{target}" + for name, target in ( + ("create", "create"), + ("list", "list"), + ("retrieve", "retrieve"), + ("content", "retrieve_content"), + ("update", "update"), + ("delete", "delete"), + ) + }, + "acreate_interaction": "interactions.acreate", + "aget_interaction": "interactions.aget", + "adelete_interaction": "interactions.adelete", + "acancel_interaction": "interactions.acancel", + } +) + +_SYNC_GENERIC: Final[Mapping[str, str]] = MappingProxyType( + { + "generate_content": "google_genai.generate_content", + "generate_content_stream": "google_genai.generate_content_stream", + "ocr": "ocr.ocr", + # The sync wrapper routes a search tool name as a model group, unlike `asearch`. + "search": "search.search", + "decisions": "decisions.decisions", + "systemone": "systemone.systemone", + "extraction": "extraction.extraction", + **{f"video_{name}": f"videos.video_{name}" for name in ("generation", "list", "status", "content", "remix")}, + **{ + name: f"containers.{name}" + for name in ("create_container", "list_containers", "retrieve_container", "delete_container") + }, + **{ + f"vector_store_{name}": f"vector_stores.main.{name}" + for name in ("search", "create", "retrieve", "list", "update", "delete") + }, + } +) + +_CONTAINER_FILE_ROUTED: Final = frozenset( + { + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + } +) +_VIDEO_DIRECT: Final = frozenset({"avideo_create_character", "avideo_get_character", "avideo_edit", "avideo_extension"}) +_SYNC_VIDEO_UNBRANCHED: Final = frozenset( + {"video_create_character", "video_get_character", "video_edit", "video_extension"} +) +_SYNC_CONTAINER_FILES_UNBRANCHED: Final = frozenset( + { + "upload_container_file", + "list_container_files", + "retrieve_container_file", + "delete_container_file", + "retrieve_container_file_content", + } +) +_UNBRANCHED: Final = frozenset( + { + "aadapter_generate_content", + *_SYNC_VIDEO_UNBRANCHED, + *_SYNC_CONTAINER_FILES_UNBRANCHED, + "get_interaction", + "delete_interaction", + "cancel_interaction", + } +) +_ASSISTANTS: Final = frozenset( + { + "acreate_assistants", + "adelete_assistant", + "aget_assistants", + "acreate_thread", + "aget_thread", + "a_add_message", + "aget_messages", + "arun_thread", + } +) +_MANAGED_AGENTS: Final[Mapping[str, str]] = MappingProxyType( + { # comprehension-ok: a flat walk of nested rows + f"{prefix}{name}": f"interactions.agents.{prefix}{target}" + for prefix in ("a", "") + for name, target in ( + ("create_agent", "create"), + ("list_agents", "list"), + ("get_agent", "get"), + ("delete_agent", "delete"), + ("list_agent_versions", "list_versions"), + ) + } +) +_SYNC_VECTOR_STORE_FILES: Final[Mapping[str, str]] = MappingProxyType( + { + f"vector_store_file_{name}": f"vector_store_files.main.{target}" + for name, target in ( + ("create", "create"), + ("list", "list"), + ("retrieve", "retrieve"), + ("content", "retrieve_content"), + ("update", "update"), + ("delete", "delete"), + ) + } +) + + +def plan_for(endpoint: str) -> Plan: + plan: Final = generic_plan(endpoint) + if plan is None: + raise KeyError(endpoint) + return plan + + +def generic_plan(endpoint: str) -> Plan | None: + if endpoint in PLANS: + return PLANS[endpoint] + if endpoint in _HOST_GENERIC: + return _generic(endpoint, _HOST_GENERIC[endpoint]) + if endpoint in _VIDEO_DIRECT: + return _generic(endpoint, f"videos.{endpoint}") + if endpoint in _SYNC_GENERIC: + return _generic(endpoint, _SYNC_GENERIC[endpoint], sync=True) + return None + + +# --- preparing the request ------------------------------------------------------------------- + + +def prepare_request( + plan: Plan, request: Mapping[str, object], model: str, aliases: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the call's kwargs + """What the router adds once before the first attempt (`_update_kwargs_before_fallbacks`): a + trace id and the requested group, in the caller's own metadata dict.""" + prepared: Final = dict(request) + if plan.kind == "vector_store_create": + return prepared # routed straight to a deployment, with no group recorded + if plan.prepare_field is None: + metadata_bucket(prepared, "metadata").update({"model_group": model}) + return prepared + prepared.setdefault("litellm_trace_id", str(uuid.uuid4())) + metadata_bucket(prepared, plan.prepare_field).update( + {"model_group": model, "model_group_alias": model if model in aliases else None} + ) + return prepared + + +# --- one attempt ---------------------------------------------------------------------------- + + +@dataclasses.dataclass(slots=True) +class AttemptCall: + """One attempt's host call: the op, its kwargs, and the request kwargs the router kept (the + ones its response headers are read from).""" + + host_op: str + kwargs: Mapping[str, object] + served: Mapping[str, object] + model_name: str + group: str + # `_embedding`'s batch-size refusal, raised by the driver in place of the host call. + refused: Exception | None = None + + +def reports_own_success(call: AttemptCall) -> bool: + """A realtime attempt's return says nothing about the session (a dropped one returns too), so + the host counts its success itself (`clean_realtime_session_deployment`).""" + return call.host_op == "_arealtime" + + +def clean_realtime_session_deployment(call: AttemptCall) -> str | None: + """`_ageneric_api_call_with_fallbacks`: the deployment a finished `_arealtime` session counts a + success against, when its success handlers ran and it did not end abnormally.""" + from litellm.constants import REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + session_logging: Final = call.kwargs.get("litellm_logging_obj") + if not reports_own_success(call) or not isinstance(session_logging, LiteLLMLogging): + return None + details: Final = session_logging.model_call_details + if details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is not True: + return None + if details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is True: + return None + deployment_id: Final = as_dict(call.kwargs.get("model_info")).get("id") + return None if deployment_id is None else str(deployment_id) + + +# `_scheduler` once `del router.scheduler` ran (None is a scheduler not built yet). +_NO_SCHEDULER: Final = object() + + +class Counters: + """`total_calls`, `success_calls` and `fail_calls`, keyed as each endpoint keys them.""" + + __slots__ = ("fail_calls", "success_calls", "total_calls") + + def __init__(self) -> None: + self.total_calls: defaultdict[str, int] = defaultdict(int) # mutable-ok: the router's live counters + self.success_calls: defaultdict[str, int] = defaultdict(int) # mutable-ok: same + self.fail_calls: defaultdict[str, int] = defaultdict(int) # mutable-ok: same + + @staticmethod + def _key(which: CountKey, model_name: str, group: str) -> str: + return model_name if which == "deployment" else group if which == "group" else "" + + def started(self, plan: Plan, call: AttemptCall) -> None: + if plan.total_key is not None: + self.total_calls[self._key(plan.total_key, call.model_name, call.group)] += 1 + + def succeeded(self, plan: Plan, call: AttemptCall) -> None: + if plan.success_key is not None: + self.success_calls[self._key(plan.success_key, call.model_name, call.group)] += 1 + + def retract_success(self, plan: Plan, call: AttemptCall) -> None: + """A served response the router then raised over: the attempt helper never reached its count.""" + if plan.success_key is not None: + self.success_calls[self._key(plan.success_key, call.model_name, call.group)] -= 1 + + def failed(self, plan: Plan, model_name: str | None, group: str) -> None: + if plan.fail_key is None: + return + if plan.fail_key == "deployment" and model_name is None: + return + self.fail_calls[self._key(plan.fail_key, model_name or "", group)] += 1 + + +def _compaction_surface(core: EndpointCore, plan: Plan) -> Surface | None: + """``surface_for_call`` over the function the Python router's fallback loop is handed.""" + if plan.kind == "generic" and surface_for_call(_handler_name(plan.host_op)) is not None: + return surface_for_call(invoked_handler_name(core, plan) or "") + return "chat" if plan.kind == "chat" and not plan.sync else None + + +def arm_compaction_state( + core: EndpointCore, + plan: Plan, + request: dict[str, object], # mutable-ok: the request kwargs, written in place as the Python router writes them +) -> None: + """``initialize_compaction_state`` at the top of ``async_function_with_fallbacks``.""" + surface: Final = _compaction_surface(core, plan) + if surface is not None: + request["_context_compaction_state"] = initialize_compaction_state(request, surface) # rebind-ok: its own + + +async def compacted_attempt( + core: EndpointCore, plan: Plan, deployment: Mapping[str, object], call: "AttemptCall" +) -> Mapping[str, object]: + """The attempt's kwargs after ``compact_to_fit``, which ``_acompletion`` and the generic helper + run on every attempt: it is also what refuses a compactor's own oversized child call.""" + if plan.kind != "generic" and (plan.kind != "chat" or plan.sync): + return call.kwargs + ctx: Final = RoutingContext(core.router) # pyright: ignore[reportArgumentType] # the checks read registry(), settings() and the token count + return await compact_to_fit(ctx, deployment, call.kwargs, _compaction_surface(core, plan)) + + +def attempt_call( + core: EndpointCore, + plan: Plan, + request: Mapping[str, object], + deployment: Mapping[str, object], + *, + group: str, + requested: str, + fallback_level: int = 0, +) -> AttemptCall: + """The host call for one attempt of ``plan`` against ``deployment``: the request (as the attempt + helper receives it), shaped for the deployment and merged with its litellm params.""" + shaping: Final = core.shaping + kwargs: Final = without(request, *ROUTING_KEYS) + named: Final = {name: kwargs.pop(name) for name in plan.named if name in kwargs} + data: Final = as_dict(deployment.get("litellm_params")) + model_name: Final = str(data.get("model")) + cache: Final = core.routing_cache + caching: Final = shaping.cache_responses + # A fallback hop routes the target group in a copy of the metadata (`run_async_fallback`). + if fallback_level > 0: + field: Final = plan.prepare_field or "metadata" + kwargs[field] = {**as_dict(kwargs.get(field)), "model_group": requested} + match plan.kind: + case "chat": + host = core.host # rebind-ok: rebound as the Python router rebinds it + drop_unsupported_classifier_reasoning_effort(deployment, requested, kwargs, host) + params = ( # rebind-ok: rebound as the Python router rebinds it + deployment_params_with_request_reasoning_override(data, kwargs) + ) + silent_model = params.pop("silent_model", None) # rebind-ok: rebound as the Python router rebinds it + start_silent_experiments(core, plan, silent_model, named.get("messages"), kwargs) + if "messages" in named: + kwargs.setdefault("messages", named["messages"]) + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache, splits_stream_timeout=True) + kwargs.pop("silent_model", None) + final = without_router_only_kwargs( # rebind-ok: rebound as the Python router rebinds it + { + **params, + **named, + "caching": caching, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + ) + return AttemptCall(plan.host_op, final, kwargs, str(params.get("model")), group) + case "text" | "embedding" | "image" | "transcription" | "rerank" | "adapter": + update_kwargs_with_deployment( + shaping, deployment, kwargs, cache=cache, splits_stream_timeout=plan.splits_stream_timeout + ) + refused: Final = embedding_batch_refusal(plan, deployment, named.get("input"), kwargs, requested) + if refused is not None: + return AttemptCall(plan.host_op, kwargs, kwargs, model_name, group, refused) + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + **named, + "caching": caching, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + return AttemptCall(plan.host_op, without_router_only_kwargs(final), kwargs, model_name, group) + case "speech": + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache) + voice = kwargs.pop("voice", None) # rebind-ok: rebound as the Python router rebinds it + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + **named, + "voice": data.get("voice") if voice is None else voice, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + return AttemptCall(plan.host_op, final, kwargs, model_name, group) + case "batch" | "cancel_batch" | "vector_store_create": + if plan.kind == "cancel_batch": + data.update(_resolved_credentials(core, deployment, requested)) + data.pop("litellm_credential_name", None) + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache, function_name=plan.function_name) + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + "custom_llm_provider": _provider(core, data), + "caching": caching, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + return AttemptCall(plan.host_op, final, kwargs, str(data.get("model")), group) + case "generic": + return _generic_attempt(core, plan, kwargs, deployment, data, requested, group) + case "generic_sync": + update_kwargs_with_deployment( + shaping, + deployment, + kwargs, + cache=cache, + function_name="generic_api_call", + splits_stream_timeout=invoked_handler_name(core, plan) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + ) + endpoint = kwargs.get("endpoint") # rebind-ok: rebound as the Python router rebinds it + if endpoint and isinstance(endpoint, str): + kwargs["endpoint"] = endpoint.replace(requested, model_name) + kwargs["model"] = model_name + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + "custom_llm_provider": _generic_provider(data), + "caching": caching, + **kwargs, + } + return AttemptCall(plan.host_op, final, kwargs, model_name, group) + case "moderation": + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache) + kwargs["model"] = model_name + kwargs.update(data) + return AttemptCall(plan.host_op, kwargs, kwargs, model_name, group) + case "search": + return _search_attempt(core, plan, kwargs, group) + case "create_file": + # `_acreate_file` creates the file on every healthy deployment of the hop's name, not + # only the picked one: the driver hands the request to `create_file_everywhere`. + return AttemptCall(plan.host_op, kwargs, kwargs, requested, group) + + +def _generic_attempt( + core: EndpointCore, + plan: Plan, + kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs + deployment: Mapping[str, object], + data: dict[str, object], # mutable-ok: the attempt's copy of the deployment params + requested: str, + group: str, +) -> AttemptCall: + """`_ageneric_api_call_with_fallbacks_helper` for one deployment.""" + from litellm.llms.base_llm.passthrough.transformation import replace_path_segment + from litellm.router_rust.invoke.websocket import turn_admission + + handler: Final = invoked_handler_name(core, plan) + socket: Final = handler == "_aresponses_websocket" + update_kwargs_with_deployment( + core.shaping, + deployment, + kwargs, + cache=core.routing_cache, + function_name="_ageneric_api_call_with_fallbacks", + splits_stream_timeout=handler in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + opens_responses_socket=socket, + ) + model_name: Final = str(data.get("model")) + endpoint: Final = kwargs.get("endpoint") + if endpoint: + try: + stripped, _, _, _ = core.host.invoke("get_llm_provider")( + model=model_name, + custom_llm_provider=kwargs.get("custom_llm_provider"), + api_base=kwargs.get("api_base"), + ) + replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it + except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is + replacement = model_name # rebind-ok: rebound as the Python router rebinds it + kwargs["endpoint"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + replace_path_segment(str(endpoint), requested, replacement) + ) + provider: Final = _generic_provider(data) + final: Final = { + **without_retrieve_stream_defaults(data, kwargs), + "caching": core.shaping.cache_responses, + **kwargs, + "model": model_name, + **_session_with_model(kwargs.get("session"), model_name), + **({"turn_admission": turn_admission(core, requested, deployment, kwargs)} if socket else {}), + } + if provider is not None: + final["custom_llm_provider"] = provider + return AttemptCall(plan.host_op, final, kwargs, model_name, group) + + +_SEARCH_PROTECTED_PARAMS: Final = frozenset({"search_provider", "api_key", "api_base"}) + + +def _search_attempt( + core: EndpointCore, + plan: Plan, + kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs + group: str, +) -> AttemptCall: + """`SearchAPIRouter.async_search_with_fallbacks_helper`: one of the search tools named + ``group`` at random, called with its provider, credentials and params under the request's.""" + import random + + tools: Final = [tool for tool in core.endpoints.search_tools if as_dict(tool).get("search_tool_name") == group] + if not tools: + raise ValueError(f"Search tool '{group}' not found in router.search_tools") + selected: Final = random.choice(tools) # load balancing, as the Python router picks + params: Final = as_dict(as_dict(selected).get("litellm_params")) + search_provider: Final = params.get("search_provider") + if not search_provider: + raise ValueError(f"search_provider not found in litellm_params for search tool '{group}'") + search_params: Final = { + key: value + for source in (params, kwargs) + for key, value in source.items() + if key not in _SEARCH_PROTECTED_PARAMS and value is not None + } # comprehension-ok: the tool's params, then the request's + final: Final = { + "search_provider": search_provider, + "api_key": params.get("api_key"), + "api_base": params.get("api_base"), + **search_params, + } + return AttemptCall(plan.host_op, final, kwargs, group, group) + + +def unrouted_fields(plan: Plan) -> Mapping[str, object]: + """A search tool name is no model group: the Rust core hands every attempt to the host as is.""" + return {"unrouted": True} if plan.kind == "search" else {} + + +def failover_fields(plan: Plan) -> Mapping[str, object]: + """An endpoint whose attempt stamps no ``failed_deployment_id`` never runs weighted failover.""" + return {} if plan.stamps_failed_deployment else {"skip_weighted_failover": True} + + +# The Python router's handler names for the plans whose `function_name` differs from it. +_PYTHON_HANDLER_NAMES: Final[Mapping[str, str]] = MappingProxyType({"acreate_file": "_acreate_file"}) + + +def scoped_fallback_fields(plan: Plan, request: Mapping[str, object]) -> Mapping[str, object]: + """A request that creates or names a resource only one provider holds (a file upload, a batch + on an uploaded file, a fine-tuning job) falls back within its own model group only + (``run_async_fallback``'s ``same_model_group_only``), decided by the Python router's own rules.""" + from litellm.router_utils.fallback_event_handlers import ( + creates_provider_scoped_resource, + references_provider_scoped_resource, + ) + + function_name: Final = plan.function_name or plan.host_op + handlers: Final = { + "original_function": _Handler(_PYTHON_HANDLER_NAMES.get(function_name, function_name)), + "original_generic_function": _Handler(plan.host_op), + } + kwargs: Final = {**request, **handlers} + scoped: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource(kwargs) + return {"same_group_fallbacks_only": True} if scoped else {} + + +@dataclasses.dataclass(frozen=True, slots=True) +class _Handler: + """A stand-in for a handler function: the scoped-resource rules read only its name.""" + + __name__: str + + +def _session_with_model(session: object, model_name: str) -> Mapping[str, object]: + if not isinstance(session, Mapping): + return {} + typed: Final = as_dict(session) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + if "model" not in typed and typed.get("type") != "transcription": + return {} + from litellm.realtime_api.main import with_resolved_session_model + + return {"session": with_resolved_session_model(typed, model_name)} + + +def _generic_provider(data: Mapping[str, object]) -> str | None: + from litellm.litellm_core_utils.routing_common_utils import provider_for_generic_call + + return provider_for_generic_call(data) # pyright: ignore[reportArgumentType] # deployment params + + +def model_client( + core: EndpointCore, plan: Plan, deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> object: + """The SDK client cached for the deployment (`_get_async_openai_model_client`; `_get_client` + with the sync keys for sync completion and embedding), dropped when the request carries + another api key.""" + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + + if as_dict(deployment.get("litellm_params")).get("kubernetes_pod_discovery") is True: + return None + model_id: Final = as_dict(deployment.get("model_info")).get("id") + sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding") + stream: Final = "stream_" if kwargs.get("stream") is True else "" + key: Final = f"{model_id}_{stream}client" if sync_keys else f"{model_id}_{stream}async_client" + span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) + client: Final = _cached_client(core, key, span, local_only=not sync_keys) + dynamic_api_key: Final = kwargs.get("api_key") + if dynamic_api_key is not None and client is not None and dynamic_api_key != getattr(client, "api_key", None): + return None + return client + + +def _cached_client(core: EndpointCore, key: str, span: object, *, local_only: bool) -> object: + cache: Final = core.routing_cache + with service_target(ROUTER_CLIENTS_TARGET): + if local_only: + return cache.get_cache(key=key, local_only=True, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # the cache holds untyped values + return cache.get_cache(key=key, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # see above + + +def _provider(core: EndpointCore, data: Mapping[str, object]) -> object: + """The deployment's declared provider, else the one its model names (`get_llm_provider`).""" + declared: Final = data.get("custom_llm_provider") + _, inferred, _, _ = core.host.invoke("get_llm_provider")(model=data["model"], custom_llm_provider=declared) + return declared or inferred + + +def _resolved_credentials(core: EndpointCore, deployment: Mapping[str, object], requested: str) -> Mapping[str, object]: + """`get_deployment_credentials_with_provider` for the picked deployment (`_acancel_batch`).""" + registry: Final = core.registry() + model_id: Final = as_dict(deployment.get("model_info")).get("id") + resolved: Final = registry.credentials(str(model_id or requested), form="with_provider") + return as_dict(resolved) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + + +def embedding_batch_refusal( + plan: Plan, deployment: Mapping[str, object], input: object, kwargs: Mapping[str, object], model: str +) -> Exception | None: + """``_embedding``'s check, past the deployment's kwargs shaping: an input the deployment admits + drops the mark an earlier rejected attempt left on the shared logging object.""" + if plan.kind != "embedding": + return None + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.router_utils.embedding_batch_size_check import ( + clear_rejected_before_provider_call, + embedding_batch_too_large_error, + ) + + error: Final = embedding_batch_too_large_error( + deployment=deployment, + input=input, + request_kwargs=kwargs, + model=model, # pyright: ignore[reportArgumentType] # opaque values + ) + logging_obj: Final = kwargs.get("litellm_logging_obj") + if error is None and isinstance(logging_obj, LiteLLMLogging): + clear_rejected_before_provider_call(logging_obj) + return error + + +def log_refused_call_sync(call: AttemptCall, refused: Exception) -> None: + """``_embedding`` logs its refusal before raising it: no provider call will.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.router_utils.embedding_batch_size_check import log_rejection_before_provider_call + + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLogging): + log_rejection_before_provider_call(logging_obj, refused) + + +async def log_refused_call(call: AttemptCall, refused: Exception) -> None: + """``log_refused_call_sync`` for ``_aembedding``.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.router_utils.embedding_batch_size_check import async_log_rejection_before_provider_call + + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLogging): + await async_log_rejection_before_provider_call(logging_obj, refused) + + +def stamp_error( + plan: Plan, error: BaseException, deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> None: + """What each attempt helper stamps on the error it raises.""" + match plan.stamp: + case "deployment": + stamp_deployment_num_retries(error, deployment) + stamp_failed_deployment_id(error, kwargs.get("model_info") or deployment.get("model_info")) + case "retry_skip": + stamp_retry_skip_deployment_id(error, kwargs) + case "none": + pass + if plan.stamp != "deployment" and plan.stamps_failed_deployment: + # `_stamp_failed_attempt`: the deployment `_update_kwargs_with_deployment` put on the kwargs. + stamp_failed_deployment_id(error, kwargs.get("model_info")) + + +# --- silent experiments --------------------------------------------------------------------- + + +_SilentModel: TypeAlias = str | list[str] +_SILENT_MODEL_ADAPTER: Final[TypeAdapter[_SilentModel]] = TypeAdapter(_SilentModel) + + +def _silent_targets(silent_model: object) -> tuple[str, ...]: + """`_silent_experiment_targets`: none inside a mirrored call or for a malformed value.""" + from litellm._internal_context import in_mirrored_call + + if silent_model is None or in_mirrored_call(): + return () + try: + targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) + except ValidationError: + verbose_router_logger.warning( + "silent_model must be a model name or a list of model names, got %r; skipping shadow traffic", + silent_model, + ) + return () + return (targets,) if isinstance(targets, str) else tuple(targets) + + +def _silent_kwargs(kwargs: Mapping[str, object]) -> dict[str, object]: # mutable-ok: the experiment's own kwargs + """`_get_silent_experiment_kwargs`.""" + from litellm.litellm_core_utils.core_helpers import ( + safe_deep_copy, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + + silent: Final = safe_deep_copy(dict(kwargs)) # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + original_metadata: Final = kwargs.get("metadata") + if original_metadata is not None and silent.get("metadata") is original_metadata: # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + silent["metadata"] = dict(original_metadata) # pyright: ignore[reportArgumentType, reportCallIssue] # the request's metadata dict + if "metadata" not in silent: + silent["metadata"] = {} + metadata: Final = silent["metadata"] # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + if isinstance(metadata, dict): + metadata.pop("litellm_parent_otel_span", None) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict + metadata["is_silent_experiment"] = True + silent.pop("litellm_call_id", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + silent.pop("litellm_logging_obj", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + silent.pop("standard_logging_object", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + return silent # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + + +def _snapshot(kwargs: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a snapshot handed to a background task + snapshot: Final = dict(kwargs) + metadata: Final = snapshot.get("metadata") + if isinstance(metadata, Mapping): + snapshot["metadata"] = dict(as_dict(metadata)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + return snapshot + + +async def _run_silent(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: + from litellm._internal_context import mirrored_call + + remaining: Final = without(kwargs, "stream") + with mirrored_call(): + response: Final = await core.acompletion( + model=silent_model, messages=messages, stream=bool(kwargs.get("stream", False)), **remaining + ) + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass + + +async def _silent_async(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: + try: + if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False): + return + silent: Final = _silent_kwargs(kwargs) + as_dict_metadata: Final = silent["metadata"] + if isinstance(as_dict_metadata, dict): + as_dict_metadata["model_group"] = silent_model + await _run_silent(core, silent_model, copy.deepcopy(messages), silent) + except Exception: # noqa: BLE001, S110 # a silent experiment never fails the primary call + pass + + +def _silent_thread(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: + try: + if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False): + return + silent: Final = _silent_kwargs(kwargs) + metadata: Final = silent["metadata"] + if isinstance(metadata, dict): + metadata["model_group"] = silent_model + loop: Final = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + + async def run() -> None: + await _run_silent(core, silent_model, copy.deepcopy(messages), silent) + pending: Final = asyncio.all_tasks() + current: Final = asyncio.current_task() + if current is not None: + pending.discard(current) + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + loop.run_until_complete(run()) + finally: + loop.close() + except Exception: # noqa: BLE001, S110 # a silent experiment never fails the primary call + pass + + +def start_silent_experiments( + core: EndpointCore, plan: Plan, silent_model: object, messages: object, kwargs: Mapping[str, object] +) -> None: + """Mirror the call to each silent model in the background: a task on the async path, a daemon + thread on the sync one.""" + for target in _silent_targets(silent_model): + if plan.sync: + threading.Thread( + target=_silent_thread, args=(core, target, messages, _snapshot(kwargs)), daemon=True + ).start() + else: + _ = asyncio.create_task( + _silent_async(core, target, messages, _snapshot(kwargs)) + ) # fire and forget, as the Python router + + +# --- the endpoints ---------------------------------------------------------------------------- + +_KEEPS_CLIENT: Final = frozenset( + { + "acreate_realtime_client_secret", + "arealtime_calls", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", + } +) +_KEEPS_PROVIDER_AND_CLIENT: Final = frozenset({"afile_delete", "afile_content"}) +_RESPONSE_ID_ENDPOINTS: Final = frozenset( + {"aget_responses", "acancel_responses", "acompact_responses", "adelete_responses", "alist_input_items"} +) +_CONTAINER_ENDPOINTS: Final = frozenset( + {"acreate_container", "alist_containers", "aretrieve_container", "adelete_container", *_CONTAINER_FILE_ROUTED} +) +_VECTOR_STORE_ENDPOINTS: Final = frozenset( + { + *(f"avector_store_{name}" for name in ("search", "create", "retrieve", "list", "update", "delete")), + *(f"avector_store_file_{name}" for name in ("create", "list", "retrieve", "content", "update", "delete")), + } +) +_INTERACTION_ENDPOINTS: Final = frozenset( + {"acreate_interaction", "aget_interaction", "adelete_interaction", "acancel_interaction"} +) +_MISSING: Final = object() + + +def pinned_video_deployment(plan: Plan, request: Mapping[str, object]) -> Mapping[str, object]: + """`_video_call_pin`: the deployment a video id names, for a call on that video only.""" + from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES, decode_video_id_with_provider + + video_id: Final = request.get("video_id") + if plan.endpoint not in VIDEO_ID_CALL_TYPES or not isinstance(video_id, str): + return {} + deployment_id: Final = decode_video_id_with_provider(video_id).get("deployment_id") + return {"pinned_deployment_id": deployment_id} if deployment_id else {} + + +async def _await(value: object) -> object: + """``await value``, raising the TypeError Python raises for a value that is not awaitable.""" + return await value # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType] # deliberately awaited as the Python router does + + +def _missing_model() -> TypeError: + return TypeError("_ageneric_api_call_with_fallbacks() missing 1 required positional argument: 'model'") + + +class RustEndpoints: + """The public endpoint methods of one Rust core, beyond the routed call loop the core drives.""" + + def __init__(self, core: EndpointCore, config: "RouterConfig") -> None: + from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout + from litellm.router_utils.request_priority import InvalidPriority, parse_default_priority + + self._core: Final = core + self.counters: Final = Counters() + explicit: Final = config.get("timeout") + self._explicit_timeout: Final = explicit + self._request_timeout: Final = get_configured_request_timeout() if explicit is not None else None + self._stream_timeout: Final = config.get("stream_timeout") + # apply() flips it after construction, as on the Python core + self._cache_responses: bool = bool(config.get("cache_responses")) + self._assistants_config: Final = ( + as_dict(config.get("assistants_config")) if config.get("assistants_config") else None + ) + # These three are assigned by apply() after construction, as on the Python core. + tools: Final = list(_as_items(config.get("search_tools"))) + self._search_tools: list[object] = tools # mutable-ok: the router's live list + self._guardrails: list[object] = list(_as_items(config.get("guardrail_list"))) # mutable-ok: same + parsed: Final = parse_default_priority(config.get("default_priority")) + self._default_priority: Final = None if isinstance(parsed, InvalidPriority) else parsed + self._polling_interval: Final = config.get("polling_interval") + self._caching_groups: Final = config.get("caching_groups") + self._default_max_parallel_requests: object = config.get("default_max_parallel_requests") + self._scheduler: object = None + + @property + def cache_responses(self) -> bool: + return self._cache_responses + + def set_cache_responses(self, value: bool) -> None: + """``cache_responses`` assigned after construction (the proxy turns it on once its cache + exists): later calls pass ``caching=value``, as the Python core's flag does.""" + self._cache_responses = value + + def assign_setting(self, name: str, value: object) -> None: + """``search_tools``, ``guardrail_list`` or ``default_max_parallel_requests`` assigned after + construction, kept as given: later searches, guardrail picks and slot limits built from + then on read it, as the Python core's attributes are read.""" + if name == "search_tools": + self._search_tools = value # pyright: ignore[reportAttributeAccessIssue] # stored as given, as the Python core stores it + elif name == "guardrail_list": + self._guardrails = value # pyright: ignore[reportAttributeAccessIssue] # same + elif name == "default_max_parallel_requests": + self._default_max_parallel_requests = value + else: + raise ValueError(f"not an endpoint setting: {name}") + + @property + def default_max_parallel_requests(self) -> object: + """The setting as given: ``settings()`` reports it and the slot limits read it.""" + return self._default_max_parallel_requests + + @property + def search_tools(self) -> list[object]: # mutable-ok: the router's live list, as the Python router reports it + return self._search_tools + + @property + def guardrail_list(self) -> list[object]: # mutable-ok: same + return self._guardrails + + @staticmethod + def prepare_host(config: "RouterConfig", host: "PythonHost") -> None: + """`cache_responses` sets the process cache, unless one is set already (the proxy's).""" + if not config.get("cache_responses") or host.global_cache() is not None: + return + from litellm.caching.caching import Cache + + redis_url: Final = config.get("redis_url") + redis_host: Final = config.get("redis_host") + redis_port: Final = config.get("redis_port") + cache_config: Final[dict[str, object]] = {} # mutable-ok: the request kwargs the Python router mutates in place + cache_type: str = "local" # rebind-ok: redis when the router is given one + if redis_url is not None or (redis_host is not None and redis_port is not None): + cache_type = "redis" # rebind-ok: rebound as the Python router rebinds it + if redis_url is not None: + cache_config["url"] = redis_url + if redis_host is not None: + cache_config["host"] = redis_host + if redis_port is not None: + cache_config["port"] = str(redis_port) + if config.get("redis_password") is not None: + cache_config["password"] = config.get("redis_password") + if config.get("redis_db") is not None: + cache_config["db"] = str(config.get("redis_db")) + cache_config.update(as_dict(config.get("cache_kwargs"))) + host.set_global_cache(Cache(type=cache_type, **cache_config)) # pyright: ignore[reportArgumentType] # the Python router's cache config + + def shaping(self, resolved: "Resolved") -> Shaping: + return Shaping( + default_litellm_params=self.default_litellm_params(resolved), + timeout=resolved.timeout, + explicit_timeout=self._explicit_timeout, + request_timeout=self._request_timeout, + stream_timeout=self._stream_timeout, + cache_responses=self._cache_responses, + ) + + def default_litellm_params(self, resolved: "Resolved") -> Mapping[str, object]: + """The defaults the Python constructor completes before any call: the router ``timeout`` + argument, no client retries, and the caching groups in the default metadata.""" + configured: Final = resolved.default_litellm_params + metadata: Final = configured.get("metadata") + return MappingProxyType( + { + "timeout": self._explicit_timeout, + "max_retries": 0, + **configured, + "metadata": { + **(as_dict(metadata) if isinstance(metadata, Mapping) else {}), # pyright: ignore[reportUnknownArgumentType] # the configured metadata + "caching_groups": self._caching_groups, + }, + } + ) + + def prepare( + self, plan: Plan, request: Mapping[str, object], model: str + ) -> dict[str, object]: # mutable-ok: the call's kwargs + return prepare_request(plan, request, model, self._core.resolved.model_group_alias) + + # -- host and registry --------------------------------------------------------------------- + + def _invoke(self, op: str) -> Callable[..., object]: + return self._core.host.invoke(op) + + async def _direct(self, op: str, kwargs: Mapping[str, object]) -> object: + return await _await(self._invoke(op)(**kwargs)) + + def _group(self, name: str) -> list[Mapping[str, object]]: # mutable-ok: a fresh list + """`_core_get_model_list(model_name=name)`: the deployments of the group ``name``.""" + records: Final = self._core.handle.deployments() + return [record for record in (as_dict(r) for r in records) if record.get("model_name") == name] + + def _is_model_group(self, name: str) -> bool: + """`_core_get_model_list(model_name=name)`: an alias, a routing group or a wildcard counts.""" + return bool(self._core.group_deployments(name)) + + # -- members ------------------------------------------------------------------------------ + + def member(self, name: str) -> Callable[..., object] | None: + """The public endpoint ``name``, bound to this core, or None when the core has none.""" + if name in _UNBRANCHED: + return _answers_none + if name in _ASSISTANTS: + return self._assistants(name) + if name in _MANAGED_AGENTS: + return self._managed_agents(name) + if name in _SYNC_VECTOR_STORE_FILES: + return self._sync_vector_store_files(name) + if name == "create_interaction": + return self._factory("create_interaction", "interactions.create") + direct: Final = { + "avector_store_create": self.avector_store_create, + "acreate_file": self.acreate_file, + "acreate_batch": self.acreate_batch, + "acancel_batch": self.acancel_batch, + "aretrieve_batch": self.aretrieve_batch, + "alist_batches": self.alist_batches, + "aadapter_completion": self.aadapter_completion, + "abatch_completion": self.abatch_completion, + "abatch_completion_one_model_multiple_requests": self.abatch_completion_one_model_multiple_requests, + "abatch_completion_fastest_response": self.abatch_completion_fastest_response, + "schedule_acompletion": self.schedule_acompletion, + "priority_header_update": self.priority_header_update, + "aguardrail": self.aguardrail, + "asearch": self.asearch, + }.get(name) + if direct is not None: + return direct + if name == "responses": + return self._sync_factory(name, PLANS["responses"]) + plan: Final = generic_plan(name) + if plan is None or name in PLANS: + return None + if plan.sync: + return self._sync_factory(name, plan) + return self._factory(name, plan.host_op) + + # -- the factory wrappers ------------------------------------------------------------------ + + def _factory(self, name: str, host_op: str) -> Callable[..., Awaitable[object]]: + async def async_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the endpoint's request + ) -> object: + return await self._wrapped(name, host_op, custom_llm_provider, client, dict(kwargs)) + + async_wrapper.__name__ = name + return async_wrapper + + async def _wrapped( + self, + name: str, + host_op: str, + custom_llm_provider: str | None, + client: object, + kwargs: dict[str, object], # mutable-ok: the wrapper's own kwargs + ) -> object: + plan: Final = generic_plan(name) or _generic(name, host_op) + if name in _CONTAINER_ENDPOINTS: + return await self._containers(plan, custom_llm_provider, kwargs) + if name in _RESPONSE_ID_ENDPOINTS: + from litellm.responses.utils import ResponsesAPIRequestUtils + + model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(kwargs.get("response_id")) # pyright: ignore[reportArgumentType] # any id the caller sends + if model_id is not None: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "model" + ] = model_id + return await self._ageneric(plan, kwargs) + if name in _VECTOR_STORE_ENDPOINTS: + return await self._vector_stores(name, plan, custom_llm_provider, kwargs) + if name in _INTERACTION_ENDPOINTS or name == "create_interaction": + return await self._interactions(plan, custom_llm_provider, kwargs) + if name in _KEEPS_CLIENT: + return await self._ageneric(plan, {**kwargs, "client": client}) + if name in _KEEPS_PROVIDER_AND_CLIENT: + return await self._ageneric(plan, {**kwargs, "custom_llm_provider": custom_llm_provider, "client": client}) + if name == "allm_passthrough_route": + if client: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "client" + ] = client + return await self._ageneric(plan, kwargs, passthrough=True) + return await self._ageneric(plan, kwargs) + + async def generic(self, plan: Plan, request: Mapping[str, object]) -> object: + """A dispatched generic endpoint called with its whole request (the wrapper's + ``custom_llm_provider`` and ``client`` are consumed by the wrapper).""" + kwargs: Final = without(request, "custom_llm_provider", "client") + return await self._ageneric(plan, kwargs) + + async def _ageneric( + self, + plan: Plan, + kwargs: Mapping[str, object], + *, + passthrough: bool = False, + ) -> object: + """`_ageneric_api_call_with_fallbacks`.""" + if "model" not in kwargs: + raise _missing_model() + + async def passthrough_call(prepared: Mapping[str, object], name: str) -> object: + sent: Final = without(prepared, *ROUTING_KEYS) + return await self._direct(plan.host_op, {"model": name, **sent}) + + return await self._core.drive(plan, kwargs, passthrough=passthrough_call if passthrough else None) + + def _sync_factory(self, name: str, plan: Plan) -> Callable[..., object]: + is_vector_store: Final = name.startswith("vector_store_") + + def sync_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the endpoint's request + ) -> object: + if not is_vector_store: + if "model" not in kwargs: + raise TypeError( + "_generic_api_call_with_fallbacks() missing 1 required positional argument: 'model'" + ) + return self._core.call_sync(plan, kwargs, _SINGLE_ATTEMPT) + provider_kwargs: dict[ # mutable-ok: the request kwargs the Python router mutates in place # rebind-ok: rebound as the Python router rebinds it + str, object + ] = ( + {**kwargs, "custom_llm_provider": custom_llm_provider} + if custom_llm_provider and "custom_llm_provider" not in kwargs + else dict(kwargs) + ) + if name == "vector_store_search": + provider_kwargs = { # rebind-ok: rebound as the Python router rebinds it + **provider_kwargs, + "_direct_vector_store_embedding_executor": self._embedding_executor(kwargs), + } + if provider_kwargs.get("model"): + return self._core.call_sync(plan, provider_kwargs, _SINGLE_ATTEMPT) + if name == "vector_store_search": + return self._invoke(plan.host_op)(**{**provider_kwargs, "router": self._core.router}) + return self._invoke(plan.host_op)(**provider_kwargs) + + sync_wrapper.__name__ = name + return sync_wrapper + + def _embedding_executor(self, kwargs: Mapping[str, object]) -> object: + from litellm.llms.base_llm.vector_store.transformation import ( + RouterVectorStoreEmbeddingExecutor, + vector_store_request_metadata, + ) + + return RouterVectorStoreEmbeddingExecutor( + router=self._core.router, # pyright: ignore[reportArgumentType] # the router serves aembedding + metadata=vector_store_request_metadata(kwargs), + ) + + async def _containers( + self, + plan: Plan, + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place + ) -> object: + """`_init_containers_api_endpoints`.""" + from litellm.responses.utils import ResponsesAPIRequestUtils + + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + container_id: Final = kwargs.get("container_id") + forwarded: Final = kwargs.get("model_id") + if isinstance(container_id, str): + decoded: Final = ResponsesAPIRequestUtils.decode_container_id(container_id) + original_id: Final = decoded.get("response_id", container_id) + if original_id != container_id: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "container_id" + ] = original_id + decoded_provider: Final = decoded.get("custom_llm_provider") + if decoded_provider and kwargs.get("custom_llm_provider") == "openai": + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = decoded_provider + model_id: Final = decoded.get("model_id") or ( + forwarded.strip() if isinstance(forwarded, str) and forwarded.strip() else None + ) + if model_id: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "model" + ] = model_id + return await self._ageneric(plan, kwargs) + requested: Final = kwargs.get("model") + if isinstance(requested, str) and requested.strip(): + return await self._ageneric(plan, kwargs, passthrough=True) + return await self._direct(plan.host_op, kwargs) + + async def _vector_stores( + self, + name: str, + plan: Plan, + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: see _wrapped + ) -> object: + """`_init_vector_store_api_endpoints`.""" + if name == "avector_store_search": + kwargs = { # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + **kwargs, + "_direct_vector_store_embedding_executor": self._embedding_executor(kwargs), + } + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + if kwargs.get("model"): + return await self._ageneric(plan, kwargs) + if name == "avector_store_search": + return await self._direct(plan.host_op, {**kwargs, "router": self._core.router}) + return await self._direct(plan.host_op, kwargs) + + async def _interactions( + self, + plan: Plan, + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place + ) -> object: + """`_init_interactions_api_endpoints`.""" + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + if "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = "gemini" + if kwargs.get("agent") and kwargs.get("model") == kwargs.get("agent"): + kwargs["model"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + None + ) + if kwargs.get("model"): + return await self._ageneric(plan, kwargs) + return await self._direct(plan.host_op, kwargs) + + def _managed_agents(self, name: str) -> Callable[..., object]: + op: Final = _MANAGED_AGENTS[name] + + def prepared( + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place + ) -> dict[str, object]: # mutable-ok: the call's kwargs + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + if "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = "gemini" + return kwargs + + if name.startswith("a"): + + async def managed_agents_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return await self._direct(op, prepared(custom_llm_provider, dict(kwargs))) + + return managed_agents_wrapper + + def managed_agents_sync_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return self._invoke(op)(**prepared(custom_llm_provider, dict(kwargs))) + + return managed_agents_sync_wrapper + + def _sync_vector_store_files(self, name: str) -> Callable[..., object]: + op: Final = _SYNC_VECTOR_STORE_FILES[name] + + def vector_store_file_sync_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return self._invoke(op)(custom_llm_provider=custom_llm_provider, client=client, **kwargs) + + return vector_store_file_sync_wrapper + + def _assistants(self, name: str) -> Callable[..., Awaitable[object]]: + async def assistants_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + """`_pass_through_assistants_endpoint_factory`.""" + provider: object = custom_llm_provider # rebind-ok: the assistants config's when none is given + sent: Final = dict(kwargs) + if provider is None: + if self._assistants_config is None: + raise Exception( # noqa: TRY002 # the Python router raises a bare Exception + "'custom_llm_provider' must be set. Either via:\n `Router(assistants_config={'custom_llm_provider': ..})` \nor\n `router.arun_thread(custom_llm_provider=..)`" + ) + provider = self._assistants_config[ # rebind-ok: rebound as the Python router rebinds it + "custom_llm_provider" + ] + sent.update(as_dict(self._assistants_config.get("litellm_params"))) + return await self._direct(name, {"custom_llm_provider": provider, "client": client, **sent}) + + return assistants_wrapper + + # -- completions, priority and prompt management ---------------------------------------- + + def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: + from litellm.litellm_core_utils.routing_common_utils import get_request_team_id + from litellm.router_utils.request_priority import request_drops_params + + def deployment_params() -> tuple[Mapping[str, object], ...]: + names: Final = {as_dict(r).get("model_name") for r in self._core.handle.deployments()} + pinned: Final = ( + self._core.handle.deployment(model) if model not in names and isinstance(model, str) else None # pyright: ignore[reportUnnecessaryIsInstance] # a caller's value, typed for the common case + ) + if pinned is not None: + return (as_dict(as_dict(pinned).get("litellm_params")),) + if kwargs.get("specific_deployment") is True: + candidates = [ # rebind-ok: rebound as the Python router rebinds it + as_dict(r) + for r in self._core.handle.deployments() + if as_dict(as_dict(r).get("litellm_params")).get("model") == model + ] + else: + # `_core_get_model_list`: an alias's target, a routing group's members and a + # wildcard's deployments list here too. + candidates = [ # rebind-ok: rebound as the Python router rebinds it + as_dict(record) + for record in self._core.registry().deployments(model, team_id=get_request_team_id(kwargs)) or () + ] + return tuple(as_dict(candidate.get("litellm_params")) for candidate in candidates) + + return request_drops_params( + kwargs, + self._core.resolved.default_litellm_params, + deployment_params(), + ) + + def _split_priority( + self, model: str, kwargs: Mapping[str, object] + ) -> tuple[int | None, dict[str, object]]: # mutable-ok: the call's kwargs + """`_core_split_request_priority`.""" + from litellm.router_utils.request_priority import InvalidPriority, priority_error, resolve_request_priority + + priority: Final = resolve_request_priority( + requested=kwargs.get("priority"), + default_priority=self._default_priority, + drops_params=lambda: self._drops_params(model=model, kwargs=kwargs), + ) + if isinstance(priority, InvalidPriority): + raise priority_error(model=model, message=priority.message) + return priority, without(kwargs, "priority") + + def priority_header_update( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: + from litellm.router_utils.request_priority import priority_header_fields + + return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs)) + + def set_scheduler(self, scheduler: object) -> None: + """``router.scheduler = ...``, which the next prioritized call waits on.""" + self._scheduler = scheduler + + def delete_scheduler(self) -> None: + """``del router.scheduler``: a prioritized call raises from then on, as on the Python core.""" + self._scheduler = _NO_SCHEDULER + + def _get_scheduler(self) -> object: + if self._scheduler is _NO_SCHEDULER: + raise AttributeError("'RustCore' object has no attribute 'scheduler'", name="scheduler") + if self._scheduler is None: + from litellm.scheduler import Scheduler + + self._scheduler = ( + Scheduler(polling_interval=self._polling_interval) # pyright: ignore[reportArgumentType] # as configured + if self._polling_interval is not None + else Scheduler() + ) + return self._scheduler + + async def _schedule( + self, model: str, priority: int, run: Callable[[], Awaitable[object]], request: Mapping[str, object] + ) -> object: + """`_schedule_factory`: wait for the scheduler's turn, run, mark the response.""" + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.scheduler import FlowItem, Scheduler + + scheduler: Final = self._get_scheduler() + assert isinstance(scheduler, Scheduler) + + async def healthy_deployments() -> Sequence[object]: + healthy: Final = await self._core.select(model, dict(request), kind="healthy") + return list(healthy) if isinstance(healthy, Sequence) else [healthy] + + await scheduler.wait_for_turn( + request=FlowItem(priority=priority, request_id=str(uuid.uuid4()), model_name=model), + timeout=self._core.resolved.timeout, # pyright: ignore[reportArgumentType] # the router timeout + get_healthy_deployments=healthy_deployments, + ) + try: + response: Final = await run() + except Exception as error: + setattr(error, "priority", priority) # noqa: B010 # the Python router stamps the same attribute + raise + hidden: Final = get_hidden_params(response) + if hidden is not None: + headers: Final = hidden.setdefault("additional_headers", {}) + if isinstance(headers, dict): + headers.update({"x-litellm-request-prioritization-used": True}) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + return response + + async def acompletion(self, request: Mapping[str, object]) -> object: + model: Final = request.get("model") + if model is None and "model" in request: + return await self._core.drive(PLANS["acompletion"], request) + if "model" not in request: + raise TypeError("acompletion() needs a model name") + if not isinstance(model, str): + _ = self._core.routed_name(model) + kwargs: Final = {**request, "stream": request.get("stream", False)} + start_time: Final = time.time() + response: Final = await self._routed_acompletion(model, kwargs) # pyright: ignore[reportArgumentType] # the name as given + self._core.service_logged("acompletion", start_time, time.time(), kwargs) + return response + + async def _routed_acompletion(self, model: str, kwargs: Mapping[str, object]) -> object: + plan: Final = PLANS["acompletion"] + priority, rest = self._split_priority(model, kwargs) + from litellm.router_rust.invoke.prompt_management import ( + is_prompt_management_model, + prompt_management_acompletion, + ) + + if is_prompt_management_model(self._core, model): + prepared: Final = self.prepare(plan, rest, model) + messages: Final = rest.get("messages") + + def factory() -> Awaitable[object]: + return prompt_management_acompletion(self._core, model, messages, prepared) + + if priority is None: + return await factory() + return await self._schedule(model, priority, factory, prepared) + if priority is None: + return await self._core.drive(plan, rest) + return await self._schedule(model, priority, lambda: self._core.drive(plan, rest), rest) + + async def atext_completion(self, request: Mapping[str, object]) -> object: + model: Final = request.get("model") + if model is None and "model" in request: + return await self._core.drive(PLANS["atext_completion"], request) + if "model" not in request: + raise TypeError("atext_completion() needs a model name") + if not isinstance(model, str): + _ = self._core.routed_name(model) + kwargs: Final = without(request, "is_retry", "is_fallback", "is_async") + plan: Final = PLANS["atext_completion"] + priority, rest = self._split_priority(model, kwargs) # pyright: ignore[reportArgumentType] # the name as given + if priority is None: + return await self._core.drive(plan, rest) + return await self._schedule(model, priority, lambda: self._core.drive(plan, rest), rest) # pyright: ignore[reportArgumentType] # see above + + async def schedule_acompletion( + self, + model: str, + messages: object, + priority: int | None = None, + stream: bool = False, + **kwargs: object, # kwargs-ok: the request + ) -> object: + from litellm.router_utils.request_priority import priority_error, unqueueable_priority_message + + queued, _ = self._split_priority(model, {**kwargs, "priority": priority}) + if queued is None: + raise priority_error(model=model, message=unqueueable_priority_message(priority)) + return await self._core.acompletion(model=model, messages=messages, stream=stream, priority=queued, **kwargs) + + # -- batch completions ------------------------------------------------------------------- + + async def _no_exceptions(self, **kwargs: object) -> object: # kwargs-ok: the request + try: + return await self._core.acompletion(**kwargs) + except Exception as error: # noqa: BLE001 # returned in place, as the Python router does + return error + + async def abatch_completion( + self, + models: list[str], # mutable-ok: the request kwargs the Python router mutates in place + messages: object, + **kwargs: object, # kwargs-ok: the endpoint's open request + ) -> object: + if isinstance(messages, list) and all(isinstance(m, dict) for m in messages): # pyright: ignore[reportUnknownVariableType] # checked item by item + return await asyncio.gather( + *(self._no_exceptions(model=model, messages=messages, **kwargs) for model in models) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + ) + if isinstance(messages, list) and all(isinstance(m, list) for m in messages): # pyright: ignore[reportUnknownVariableType] # same + rows: Final[list[object]] = list(messages) # pyright: ignore[reportUnknownArgumentType] # same # mutable-ok: the request kwargs the Python router mutates in place + + async def indexed(model: str, message: object, idx: int) -> tuple[object, int]: + return await self._no_exceptions(model=model, messages=message, **kwargs), idx + + responses: Final = await asyncio.gather( + *( + indexed(model, message, idx) for idx, message in enumerate(rows) for model in models + ) # comprehension-ok: a flat walk of nested rows + ) + final: Final[list[list[object]]] = [ # mutable-ok: the request kwargs the Python router mutates in place + [] for _ in rows + ] + for response, idx in responses: + final[idx].append(response) + return final + return None + + async def abatch_completion_one_model_multiple_requests( + self, + model: str, + messages: list[object], # mutable-ok: the request kwargs the Python router mutates in place + **kwargs: object, # kwargs-ok: the endpoint's open request + ) -> object: + return await asyncio.gather(*(self._no_exceptions(model=model, messages=m, **kwargs) for m in messages)) + + async def abatch_completion_fastest_response( + self, + model: str, + messages: object, + stream: bool = False, + **kwargs: object, # kwargs-ok: the request + ) -> object: + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.types.utils import ModelResponse + + models: Final = [m.strip() for m in model.split(",")] + + async def no_exceptions(model_name: str) -> object: + try: + return await self._core.acompletion(model=model_name, messages=messages, stream=stream, **kwargs) + except asyncio.CancelledError: + raise + except Exception as error: # noqa: BLE001 # see _no_exceptions + return error + + pending: set[ # mutable-ok: the request kwargs the Python router mutates in place # rebind-ok: rebound as the Python router rebinds it + asyncio.Task[object] + ] = {asyncio.create_task(no_exceptions(m)) for m in models} + while pending: + done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + for task in done: + result = task.result() + if isinstance(result, (ModelResponse, CustomStreamWrapper)): + for other in pending: + other.cancel() + hidden = get_hidden_params(result) + if hidden is not None: + hidden["fastest_response_batch_completion"] = True + return result + raise Exception("All tasks failed") # noqa: TRY002 # the Python router raises a bare Exception + + # -- moderation, text completion and adapter -------------------------------------------- + + async def amoderation(self, request: Mapping[str, object]) -> object: + """`_pass_through_moderation_endpoint_factory`: one pick when the model is a group.""" + kwargs: Final = without(request, "custom_llm_provider", "client") + plan: Final = PLANS["amoderation"] + model: Final = kwargs.get("model", "") + if not isinstance(model, str): + _ = self._core.routed_name(model) + if model and isinstance(model, str) and self._is_model_group(model): + return await self._core.drive(plan, kwargs, _SINGLE_ATTEMPT) + prepared: Final = prepare_request(plan, kwargs, str(model), self._core.resolved.model_group_alias) + return await self._direct("amoderation", prepared) + + def text_completion(self, request: Mapping[str, object]) -> object: + """The sync `text_completion`: one pick, no retries, the group name as the model.""" + from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin + + kwargs: Final = without(request, "is_retry", "is_fallback", "is_async") + model: Final = kwargs.get("model") + prompt: Final = kwargs.get("prompt") + if "model" not in kwargs: + raise TypeError("text_completion() missing 1 required positional argument: 'model'") + kwargs["num_retries"] = kwargs.get("num_retries", self._core.resolved.num_retries) + metadata_bucket(kwargs, "metadata").update({"model_group": model}) + with endpoint_selection("text_completion", model=model): + deployment: Final = self._core.router.get_available_deployment( + model=model, # pyright: ignore[reportArgumentType] # the name as given + messages=_user_message(prompt), + specific_deployment=kwargs.pop("specific_deployment", None), # pyright: ignore[reportArgumentType] # the request's value, as given + request_kwargs=kwargs, + ) + data: Final = as_dict(deployment.get("litellm_params")) + for key, value in self._core.resolved.default_litellm_params.items(): + if key not in kwargs: + kwargs[key] = value + elif key == "metadata": + bucket = kwargs[key] + if isinstance(bucket, dict): + bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict + pinned: Final = deployment_model_id_pin(data) + if pinned is not None: + kwargs["model_id"] = pinned + return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs}) + + async def aadapter_completion( + self, + adapter_id: str, + model: str, + is_retry: bool | None = False, + is_fallback: bool | None = False, + is_async: bool | None = False, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return await self._core.drive( + PLANS["aadapter_completion"], {**kwargs, "model": model, "adapter_id": adapter_id} + ) + + # -- files, batches and vector stores --------------------------------------------------- + + async def acreate_batch(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + return await self._core.drive(PLANS["acreate_batch"], {**kwargs, "model": model}) + + async def acancel_batch(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + return await self._core.drive(PLANS["acancel_batch"], {**kwargs, "model": model}) + + async def avector_store_create(self, model: str | None, **kwargs: object) -> object: # kwargs-ok: the request + if model is None: + return await self._factory("avector_store_create", "vector_stores.main.acreate")(**kwargs) + return await self._core.drive(PLANS["avector_store_create"], {**kwargs, "model": model}, _SINGLE_ATTEMPT) + + async def aretrieve_batch(self, model: str | None = None, **kwargs: object) -> object: # kwargs-ok: the request + """Ask every deployment of the group (or of the router) for the batch.""" + from litellm.litellm_core_utils.core_helpers import ( + safe_deep_copy, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + from litellm.types.utils import LiteLLMBatch + + shaping: Final = self._core.shaping + metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch") + if model is not None: + specific: Final = kwargs.pop("specific_deployment", None) + healthy = await self._core.select( # rebind-ok: rebound as the Python router rebinds it + model, + dict(kwargs), + kind="healthy", + specific_deployment=bool(specific) or None, + messages=_user_message("retrieve-api-fake-text"), + ) + else: + healthy = [as_dict(r) for r in self._core.handle.deployments()] # rebind-ok: as Python rebinds it + errors: Final[list[BaseException]] = [] # mutable-ok: the request kwargs the Python router mutates in place + + async def try_retrieve(deployment: Mapping[str, object]) -> object: + try: + data: Final = as_dict(deployment.get("litellm_params")) + deployment_model: Final = data.get("model") + provider: object = data.get("custom_llm_provider") # rebind-ok: inferred when not declared + if deployment_model is None: + raise Exception(f"Model not found in litellm_params for deployment: {deployment}") # noqa: TRY002 # as the Python router + if not provider: + lookup: Final = self._core.host.invoke("get_llm_provider") + _, provider, _, _ = lookup(model=deployment_model) # rebind-ok: as Python rebinds it + new_kwargs: Final = safe_deep_copy(dict(kwargs)) # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + update_kwargs_with_deployment( + shaping, + deployment, + new_kwargs, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + cache=self._core.routing_cache, + function_name="aretrieve_batch", + ) + bucket: Final = metadata_bucket(new_kwargs, metadata_variable_name) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + if not bucket.get("model_group"): + bucket["model_group"] = model or deployment.get("model_name") + new_kwargs.pop("custom_llm_provider", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + data.pop("custom_llm_provider", None) + return await self._direct("aretrieve_batch", {**data, "custom_llm_provider": provider, **new_kwargs}) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + except Exception as error: # noqa: BLE001 # collected; the first one is raised when no deployment has the batch + errors.append(error) + return None + + deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy] + if not deployments: + raise Exception("No healthy deployments found.") # noqa: TRY002 # as the Python router + results: Final = await asyncio.gather(*(try_retrieve(as_dict(d)) for d in deployments), return_exceptions=True) + for result in results: + if isinstance(result, LiteLLMBatch): + return result + if errors: + raise errors[0] + raise Exception(f"Unable to find batch in any model. Received errors - {errors}") # noqa: TRY002 # as the Python router + + async def alist_batches(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + deployments: Final = self._group(model) + + async def try_list(deployment: Mapping[str, object]) -> object: + try: + return await self._direct("alist_batches", {**as_dict(deployment.get("litellm_params")), **kwargs}) + except Exception: # noqa: BLE001 # a failing deployment is left out, as the Python router does + return None + + results: Final = await asyncio.gather(*(try_list(d) for d in deployments)) + final: Final[dict[str, object]] = { # mutable-ok: the request kwargs the Python router mutates in place + "object": "list", + "data": [], + "first_id": None, + "last_id": None, + "has_more": False, + } + data: Final[list[object]] = [] # mutable-ok: the request kwargs the Python router mutates in place + final["data"] = data + for result in results: + if result is None: + continue + if final["first_id"] is None and hasattr(result, "first_id"): + final["first_id"] = getattr(result, "first_id") # noqa: B009 # as the Python router + final["last_id"] = getattr(result, "last_id") # noqa: B009 # same + data.extend(getattr(result, "data")) # noqa: B009 # same # pyright: ignore[reportAny] # litellm's untyped helpers and request values + if getattr(result, "has_more", False) is True: + final["has_more"] = True + return final + + async def acreate_file(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + return await self._core.drive(PLANS["acreate_file"], {**kwargs, "model": model}) + + async def create_file_everywhere(self, kwargs: Mapping[str, object], requested: str, specific: object) -> object: + """`_acreate_file`: create the file on every healthy deployment of the group.""" + from litellm.litellm_core_utils import batch_utils + from litellm.litellm_core_utils.core_helpers import ( + safe_deep_copy, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.litellm_core_utils.routing_common_utils import ( + add_model_file_id_mappings, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + from litellm.types.utils import LlmProviders + + healthy: Final = await self._core.select( + requested, + dict(kwargs), + kind="healthy", + specific_deployment=bool(specific) or None, + messages=_user_message(_PICK_TEXTS["create_file"]), + ) + deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy] + + async def create_for(deployment: Mapping[str, object]) -> object: + kwargs_copy: Final = safe_deep_copy(dict(kwargs)) # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + update_kwargs_with_deployment( + self._core.shaping, + deployment, + kwargs_copy, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + cache=self._core.routing_cache, + function_name="acreate_file", + ) + data: Final = as_dict(deployment.get("litellm_params")) + model_name: Final = str(data.get("model")) + self.counters.total_calls[model_name] += 1 + declared: Final = data.get("custom_llm_provider") + stripped, inferred, _, _ = self._core.host.invoke("get_llm_provider")( + model=data["model"], custom_llm_provider=declared + ) + provider: Final = declared or inferred + purpose: Final = kwargs.get("purpose") + file: object = kwargs.get("file") # rebind-ok: rewritten for the deployment + if not file or not purpose: + raise Exception("file and file_purpose are required for create_file") # noqa: TRY002 # as the Python router + replaces: Final = batch_utils.should_replace_model_in_jsonl( + purpose=purpose, # pyright: ignore[reportArgumentType] # the caller's purpose + passthrough=kwargs.get("passthrough") is True, + ) + if replaces: + file = batch_utils.replace_model_in_jsonl( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # an untyped helper # rebind-ok: rewritten for the deployment + file_content=file, # pyright: ignore[reportArgumentType] # the caller's file + new_model_name=stripped, + keep_line_numbers=provider == LlmProviders.VERTEX_AI.value, + ) + kwargs_copy["file"] = file + if provider == LlmProviders.LITELLM_PROXY.value: + kwargs_copy["extra_body"] = MappingProxyType( + {**as_dict(kwargs_copy.get("extra_body")), "target_model_names": stripped} # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # litellm's untyped helpers and request values + ) + if "gcs_bucket_name" in data: + metadata_bucket(kwargs_copy, "litellm_metadata")["gcs_bucket_name"] = data["gcs_bucket_name"] # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + from litellm.router_rust.invoke.slots import deployment_slot + + with deployment_slot(self._core, deployment, kwargs_copy): # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + response: Final = await self._direct( + "acreate_file", + { + **data, + "custom_llm_provider": provider, + "caching": self._cache_responses, + "client": model_client(self._core, PLANS["acreate_file"], deployment, kwargs_copy), # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + **kwargs_copy, + }, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + ) + self.counters.success_calls[model_name] += 1 + return response + + responses: Final = await asyncio.gather(*(create_for(as_dict(d)) for d in deployments)) + if len(responses) == 0: + raise Exception("No healthy deployments found.") # noqa: TRY002 # as the Python router + mapping: Final = add_model_file_id_mappings(healthy_deployments=deployments, responses=responses) # pyright: ignore[reportArgumentType, reportUnknownVariableType] # deployment records + returned: Final = responses[0] + hidden: Final = get_hidden_params(returned) + if hidden is not None: + hidden["model_file_id_mapping"] = mapping + return returned + + # -- search tools ------------------------------------------------------------------------- + + async def asearch( + self, + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + """`_asearch_with_fallbacks`: the search tool name (``search_tool_name``, else ``model``) + routes as the model; the factory wrapper drops ``custom_llm_provider`` and ``client``.""" + request: Final = dict(kwargs) + search_tool_name: Final = request.get("search_tool_name", request.get("model")) + if not search_tool_name: + raise ValueError("search_tool_name or model parameter is required for search") + request["model"] = search_tool_name + return await self._core.drive(PLANS["asearch"], request) + + # -- guardrails --------------------------------------------------------------------------- + + async def aguardrail( + self, + guardrail_name: str, + original_function: Callable[..., Awaitable[object]], + **kwargs: object, # kwargs-ok: the request + ) -> object: + """Pick one of the guardrails named ``guardrail_name`` (weighted) and run it. A guardrail name + is no model group, so the router's retries never apply.""" + request: Final = dict(kwargs) + request.setdefault("litellm_trace_id", str(uuid.uuid4())) + metadata_bucket(request, "litellm_metadata").update( + { + "model_group": guardrail_name, + "model_group_alias": guardrail_name + if guardrail_name in self._core.resolved.model_group_alias + else None, + } + ) + verdict: Final = await original_function(**request, selected_guardrail=self.available_guardrail(guardrail_name)) + # `make_call`, `async_function_with_retries` and `async_function_with_fallbacks` stamp the result. + headed: Final = await self._core.set_response_headers(verdict, guardrail_name, request) + return add_fallback_headers_to_response( + response=add_retry_headers_to_response(response=headed, attempted_retries=0, max_retries=None), + attempted_fallbacks=0, + ) + + def available_guardrail(self, guardrail_name: str) -> object: + """``_core_get_available_guardrail``: the one guardrail configured under the name, else one of + them picked by weight (``simple_shuffle``, the name resolved through the alias map).""" + from litellm.router_strategy.simple_shuffle import simple_shuffle + + candidates: Final = [g for g in self._guardrails if as_dict(g).get("guardrail_name") == guardrail_name] + if not candidates: + raise ValueError(f"No guardrail found with name: {guardrail_name}") + if len(candidates) == 1: + return candidates[0] + return simple_shuffle( # pyright: ignore[reportUnknownVariableType] # litellm's untyped helper + resolve_model_alias=self._core.resolve_alias, + healthy_deployments=candidates, # pyright: ignore[reportArgumentType] # guardrail configs + model=guardrail_name, + request_kwargs=None, + ) + + # -- attempts ----------------------------------------------------------------------------- + + def attempt_failed( + self, + plan: Plan, + error: BaseException, + deployment: Mapping[str, object], + call: AttemptCall | None, + requested: str, + ) -> None: + from litellm import exceptions as litellm_exceptions + from litellm.types.router import LiteLLM_Params + + # `_acompletion`'s Timeout branch counts no failure and adds the deployment's timeouts; the + # sync `_completion` has no such branch. + timed_out: Final = plan.kind == "chat" and not plan.sync and isinstance(error, litellm_exceptions.Timeout) + if not timed_out: + self.counters.failed(plan, call.model_name if call is not None else None, requested) + elif isinstance(error, litellm_exceptions.Timeout) and self._core.host.dynamic().expose_router_debug_in_errors: + params: Final = as_dict(deployment.get("litellm_params")) + try: + typed: Mapping[str, object] = ( # rebind-ok: rebound as the Python router rebinds it + LiteLLM_Params.model_validate(params).model_dump() + ) + except Exception: # noqa: BLE001 # the params as configured, when they do not validate + typed = params # rebind-ok: rebound as the Python router rebinds it + error.message += ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + f"\n\nDeployment Info: request_timeout: {typed.get('request_timeout')}" + f"\ntimeout: {typed.get('timeout')}" + f"\nstream_timeout: {typed.get('stream_timeout')}" + ) + + def check_response(self, plan: Plan, response: object, call: AttemptCall) -> None: + """What the attempt helper checks of a response before it counts it as served.""" + + def sync_pre_call_checks(self, plan: Plan, deployment: Mapping[str, object], request: Mapping[str, object]) -> None: + """`routing_strategy_pre_call_checks` on the sync path: the process callbacks' pre-call checks. + Sync completion runs them for a group but not for a deployment id.""" + from litellm.integrations.custom_logger import CustomLogger + + if plan.kind == "chat": + model: Final = str(request.get("model")) + names: Final = {as_dict(r).get("model_name") for r in self._core.handle.deployments()} + if model not in names and self._core.handle.deployment(model) is not None: + return + for callback in tuple(self._core.routing_callbacks()): + if isinstance(callback, CustomLogger): + callback.pre_call_check(dict(deployment)) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + + +_SINGLE_ATTEMPT: Final[Mapping[str, object]] = MappingProxyType({"num_retries": 0, "disable_fallbacks": True}) + + +async def _answers_none(*_args: object, **_kwargs: object) -> None: # kwargs-ok: any request + """An endpoint the Python router's async wrapper has no branch for.""" + return + + +def _as_items(value: object) -> list[object]: # mutable-ok: a fresh list + return list(value) if isinstance(value, list) else [] # pyright: ignore[reportUnknownArgumentType] # a config list + + +if TYPE_CHECKING: + from litellm.router_config import RouterConfig + from litellm.router_python_host import PythonHost + from litellm.router_rust.core import Resolved + + +__all__ = [ + "PLANS", + "ROUTING_KEYS", + "AttemptCall", + "Counters", + "Plan", + "RustEndpoints", + "Shaping", + "arm_compaction_state", + "attempt_call", + "clean_realtime_session_deployment", + "compacted_attempt", + "failover_fields", + "generic_plan", + "log_refused_call", + "log_refused_call_sync", + "pinned_video_deployment", + "plan_for", + "prepare_request", + "reports_own_success", + "selection_endpoint", + "stamp_error", + "unrouted_fields", +] diff --git a/litellm/router_rust/invoke/prompt_management.py b/litellm/router_rust/invoke/prompt_management.py new file mode 100644 index 00000000000..62bd77e1b48 --- /dev/null +++ b/litellm/router_rust/invoke/prompt_management.py @@ -0,0 +1,129 @@ +""" +Prompt-management groups on the Rust router core (``_is_prompt_management_model`` and +``_prompt_management_factory``): a group of exactly one deployment whose model names a prompt +management integration (``langfuse/...``) or ``litellm_agent/...``. The request is not routed with +retries and fallbacks: the deployment is picked, its prompt rendered by the request's logging +object, and the rendered model called directly, or routed when it names a group. +""" + +from collections.abc import Mapping +from typing import Final + +from litellm.litellm_core_utils.core_helpers import without +from litellm.router_rust.invoke.endpoints import PLANS, EndpointCore +from litellm.router_rust.invoke.shaping import as_dict, update_kwargs_with_deployment + +_PROMPT_MANAGEMENT_PARAMS: Final = frozenset( + {"bitbucket_config", "dotprompt_config", "prompt_id", "prompt_variables", "prompt_label", "prompt_version"} +) + + +def is_prompt_management_model(core: EndpointCore, model: str) -> bool: + """`_is_prompt_management_model`: one deployment, whose model's provider prefix is a known + custom-logger-compatible callback.""" + model_list: Final = core.group_deployments(model_name=model) + if model_list is None or len(model_list) != 1: + return False + litellm_model: Final = as_dict(model_list[0].get("litellm_params")).get("model") + if not isinstance(litellm_model, str) or "/" not in litellm_model: + return False + known: Final = core.host.dynamic().known_custom_logger_compatible_callbacks + return litellm_model.split("/")[0] in known + + +def _logging_and_kwargs( + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place +) -> tuple[object, dict[str, object]]: # mutable-ok: same + """The request's logging object, else the one `function_setup` builds (with the kwargs it returns).""" + from litellm.utils import Rules, function_setup, get_utc_datetime + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if logging_obj is not None: + return logging_obj, kwargs + return function_setup( # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers + **{ # pyright: ignore[reportArgumentType] # the request's keyword payload + "original_function": "acompletion", + "rules_obj": Rules(), + "start_time": get_utc_datetime(), + **kwargs, + } + ) + + +async def prompt_management_acompletion( + core: EndpointCore, + model: str, + messages: object, + request: Mapping[str, object], +) -> object: + """`_prompt_management_factory`, on the request as `acompletion` prepared it.""" + from litellm.utils import get_non_default_completion_params + + # `acompletion` hands the routing function along; function_setup consumes it. + logging_obj, kwargs = _logging_and_kwargs({**request, "original_function": core.router.acompletion}) + caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout") + caller_stream_timeout: Final = kwargs.get("stream_timeout") + specific_deployment: Final = kwargs.pop("specific_deployment", None) + picked: Final = await core.select( + model, + kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs + kind="pick", + specific_deployment=bool(specific_deployment), + messages=messages, + ) + deployment: Final = as_dict(picked) + update_kwargs_with_deployment( + core.shaping, + deployment, + kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs + cache=core.routing_cache, + splits_stream_timeout=True, + ) + params: Final = as_dict(deployment.get("litellm_params")) + data: Final = dict(params) + litellm_model: Final = data.get("model") + is_litellm_agent_model: Final = isinstance(litellm_model, str) and litellm_model.startswith("litellm_agent/") + prompt_id: Final = kwargs.get("prompt_id") or params.get("prompt_id") + prompt_variables: Final = kwargs.get("prompt_variables") or params.get("prompt_variables") + prompt_label: Final = kwargs.get("prompt_label") or params.get("prompt_label") + if not is_litellm_agent_model and (prompt_id is None or not isinstance(prompt_id, str)): + raise ValueError(f"Prompt ID is not set or not a string. Got={prompt_id}, type={type(prompt_id)}") + if prompt_variables is not None and not isinstance(prompt_variables, dict): + raise ValueError( + f"Prompt variables is set but not a dictionary. Got={prompt_variables}, type={type(prompt_variables)}" + ) + rendered_model, rendered_messages, optional_params = logging_obj.get_chat_completion_prompt( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue, reportOptionalMemberAccess] # the request's Logging object + model=litellm_model, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + messages=messages, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + non_default_params=get_non_default_completion_params(kwargs=kwargs), # pyright: ignore[reportArgumentType] # the request kwargs + prompt_id=prompt_id, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + prompt_variables=prompt_variables, + prompt_label=prompt_label, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + request_kwargs=kwargs, + injected_for_every_deployment=True, + ) + filtered_data: Final = without(data, *_PROMPT_MANAGEMENT_PARAMS) + final: Final[dict[str, object]] = {**filtered_data, **kwargs, **optional_params} # pyright: ignore[reportUnknownVariableType] # mutable-ok: the call's kwargs + final["model"] = rendered_model + final["messages"] = rendered_messages + final["litellm_logging_obj"] = logging_obj + final["prompt_id"] = prompt_id + final["prompt_variables"] = prompt_variables + final["prompt_label"] = prompt_label + target_list: Final = core.group_deployments(model_name=str(rendered_model)) # pyright: ignore[reportUnknownArgumentType] # the core's host registry + if target_list is None or len(target_list) == 0: # a direct call to the rendered model + _ = final.pop("original_function") + return await core.host.invoke("acompletion")(**final) # pyright: ignore[reportUnknownMemberType] # the core's host + timeouts: Final = { + key: value + for key, value in ( + ("timeout", caller_timeout if caller_timeout is not None else filtered_data.get("timeout")), + ( + "stream_timeout", + caller_stream_timeout if caller_stream_timeout is not None else filtered_data.get("stream_timeout"), + ), + ) + if value is not None + } + routed: Final = without(final, "timeout", "stream_timeout") + return await core.drive(PLANS["acompletion"], {**routed, **timeouts}, prepared_already=True) diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py new file mode 100644 index 00000000000..258f919c9f1 --- /dev/null +++ b/litellm/router_rust/invoke/shaping.py @@ -0,0 +1,635 @@ +""" +The keyword arguments one router attempt sends to one deployment, as the Python router builds them. + +The Rust router core picks the deployment; the host shapes the request for it here +(``_update_kwargs_with_deployment``, ``_update_kwargs_with_default_litellm_params``, +``_merge_tools_from_deployment``, the timeout resolvers and the reasoning-effort override), with +the same precedence and the same oddities. Nothing here reads a Python router. +""" + +import dataclasses +from collections.abc import Mapping, MutableMapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import TypeAdapter, ValidationError + +from litellm import exceptions as litellm_exceptions +from litellm.constants import ( + BAGGAGE_MAPPED_TAGS_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, +) +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name +from litellm.litellm_core_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + is_clientside_credential, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values +) +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.core_helpers import without +from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider +from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout +from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin +from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy +from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults +from litellm.router_api import deployment_id as generate_deployment_id +from litellm.router_strategy.tag_based_routing import ( + baggage_stamp_for_serving_deployment, + get_tags_from_request_kwargs, + tags_with_serving_deployment_tags, +) +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + deployment_has_io_token_limits, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + refund_stale_reservation_before_retry, + set_io_token_rate_limit_request_kwargs, +) +from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD +from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, validate_litellm_params_strictly +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.router_python_host import PythonHost + +# The SDK-client and max_parallel_requests reads of the routing cache: process-local, so the family +# only names a Redis span when a read misses through to Redis. +ROUTER_CLIENTS_TARGET: Final = "router_clients" +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthropic"}) +_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +# Generic handlers whose stream timeout is split from the request timeout. +STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Shaping: + """What the request shaping reads from the router, resolved once when the core is built. + + ``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``), + ``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the + router has a timeout of its own, as the Python router keeps them.""" + + default_litellm_params: Mapping[str, object] + timeout: object + explicit_timeout: object + request_timeout: float | None + stream_timeout: object + cache_responses: bool + + +def as_dict(value: object) -> dict[str, object]: # mutable-ok: a fresh dict of a request mapping + try: + return dict(_MAPPING.validate_python(value)) + except ValidationError: + return {} + + +def _mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING.validate_python(value) + except ValidationError: + return None + + +def metadata_bucket( + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + name: str, +) -> dict[str, object]: # mutable-ok: the request's own bucket + """``kwargs.setdefault(name, {})``: the request's own metadata dict, created when absent.""" + bucket: Final = kwargs.get(name) + if isinstance(bucket, dict): + return bucket # pyright: ignore[reportUnknownVariableType] # the request's untyped metadata dict + if name not in kwargs: + fresh: Final[dict[str, object]] = {} # mutable-ok: the request kwargs the Python router mutates in place + kwargs[name] = fresh # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + return fresh + # A non-dict value under the key: `setdefault` returns it as is, and `.update` would raise. + raise AttributeError(f"'{type(bucket).__name__}' object has no attribute 'update'") + + +# --- request params the deployment sets -------------------------------------------------------- + + +def _drops_malformed_request_tools( + deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] +) -> bool: + if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"): + return False + set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} + return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools") + + +def _malformed_request_params_the_deployment_sets( + deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] +) -> frozenset[str]: + overrides: Final = { + key: value + for key, value in request.items() + if key in SUPPLEMENTAL_KWARGS_KEYS and deployment_params.get(key) is not None + } + if not overrides: + return frozenset() + try: + validate_litellm_params_strictly(GenericLiteLLMParams, overrides) + except ValidationError as error: + set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} + policy: Final = drop_policy({**deployment_params, **set_defaults, **request}) + malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"]) + return frozenset(key for key in malformed if policy.drops(key)) + return frozenset() + + +def _merged_tools( + deployment_tools: Sequence[object], request_tools: object +) -> list[object] | None: # mutable-ok: sent as the request's tools + if not isinstance(request_tools, (list, tuple)): + return None + items: Final[Sequence[object]] = request_tools # pyright: ignore[reportUnknownVariableType] # a request's untyped tools + return [*deployment_tools, *items] if deployment_tools or items else None + + +def merge_tools_from_deployment( + deployment_params: Mapping[str, object], + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + defaults: Mapping[str, object], +) -> None: + """``_merge_tools_from_deployment``: deployment tools first, then the request's; the request's + ``tool_choice`` else the deployment's.""" + if _drops_malformed_request_tools(deployment_params, defaults, kwargs): + for name in TOOL_FIELDS: + kwargs.pop(name, None) + deployment_tools: Final = deployment_params.get("tools") or [] # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + merged: Final = _merged_tools( + deployment_tools if isinstance(deployment_tools, list) else [], # pyright: ignore[reportUnknownArgumentType] # the deployment's untyped tools + kwargs.get("tools") or [], + ) + if merged is not None: + kwargs["tools"] = merged # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + if "tool_choice" not in kwargs and deployment_params.get("tool_choice") is not None: + kwargs["tool_choice"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + deployment_params["tool_choice"] + ) + + +def _deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool: + value: Final = deployment_params.get(key) + if value is None: + return False + field: Final = LiteLLM_Params.model_fields.get(key) + return field is None or value != field.default # pyright: ignore[reportAny] # litellm's untyped helpers and request values + + +def _forwarded_alias_marker_keys_the_deployment_sets( + deployment_params: Mapping[str, object], forwarded_keys: object +) -> tuple[str, ...]: + if not isinstance(forwarded_keys, tuple): + return () + keys: Final[tuple[object, ...]] = forwarded_keys # pyright: ignore[reportUnknownVariableType] # the router's own marker + return tuple(key for key in keys if isinstance(key, str) and _deployment_sets_litellm_param(deployment_params, key)) + + +# --- reasoning effort ------------------------------------------------------------------------- + + +def _pop_effort_from_nested_carrier(params: dict[str, object], carrier: str) -> None: # mutable-ok: copy-on-write + nested: Final = _mapping(params.get(carrier)) if isinstance(params.get(carrier), Mapping) else None + if nested is None: + return + sanitized: Final = without(nested, "effort") + if sanitized: + params[carrier] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + sanitized + ) + else: + params.pop(carrier, None) + + +def deployment_params_with_request_reasoning_override( + deployment_params: Mapping[str, object], request_kwargs: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the call's own copy of the deployment params + """``_deployment_params_with_request_reasoning_override``: a request ``reasoning_effort`` drops + the deployment's ``thinking`` and nested ``effort`` carriers.""" + sanitized: Final = dict(deployment_params) + if request_kwargs.get("reasoning_effort") is None: + return sanitized + sanitized.pop("thinking", None) + _pop_effort_from_nested_carrier(sanitized, "output_config") + _pop_effort_from_nested_carrier(sanitized, "reasoning") + extra_body: Final = sanitized.get("extra_body") + if isinstance(extra_body, Mapping): + sanitized_extra_body: Final = as_dict(extra_body) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + sanitized_extra_body.pop("reasoning_effort", None) + sanitized_extra_body.pop("thinking", None) + _pop_effort_from_nested_carrier(sanitized_extra_body, "output_config") + _pop_effort_from_nested_carrier(sanitized_extra_body, "reasoning") + if sanitized_extra_body: + sanitized["extra_body"] = sanitized_extra_body + else: + sanitized.pop("extra_body", None) + return sanitized + + +def _is_classifier_internal_call(kwargs: Mapping[str, object]) -> bool: + return any( + isinstance(candidate, Mapping) + and (_mapping(candidate) or {}).get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + for candidate in (kwargs.get("metadata"), kwargs.get("litellm_metadata")) + ) + + +def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: + declared: Final = params.get("allowed_openai_params") + if not isinstance(declared, (list, tuple, set, frozenset)): + return frozenset() + entries: Final[Sequence[object]] = list(declared) # pyright: ignore[reportUnknownArgumentType] # the deployment's untyped allowlist + return frozenset(entry for entry in entries if isinstance(entry, str)) + + +def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: + params: Final = as_dict(deployment.get("litellm_params")) + if not params: + return True + model_info: Final = as_dict(deployment.get("model_info")) + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model_label: Final = str(params.get("model") or group) + provider_param: Final = params.get("custom_llm_provider") + authenticating: Final = declared_authenticating_provider( + model_label, provider_param if isinstance(provider_param, str) else None + ) + if authenticating is not None: + model, custom_llm_provider = model_label.removeprefix(f"{authenticating}/"), authenticating + else: + model, custom_llm_provider, _, _ = host.invoke( # rebind-ok: rebound as the Python router rebinds it + "get_llm_provider" + )( + model=model_label, + custom_llm_provider=provider_param, + litellm_params=LiteLLM_Params.model_validate(params), + ) + supported: Final = host.invoke("get_supported_openai_params")( + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, + ) + except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + return True + return supported is None or param in supported + + +def deployment_accepts_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: + params: Final = as_dict(deployment.get("litellm_params")) + if not params: + return True + if param in _declared_param_allowlist(params): + return True + provider_param: Final = params.get("custom_llm_provider") + if declared_authenticating_provider( + str(params.get("model") or ""), provider_param if isinstance(provider_param, str) else None + ): + return True + return _deployment_supports_param(deployment, group, param, host) + + +def drop_unsupported_classifier_reasoning_effort( + deployment: Mapping[str, object], + model: str, + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + host: "PythonHost", +) -> None: + """``_drop_unsupported_classifier_reasoning_effort``.""" + if kwargs.get("reasoning_effort") is None or not _is_classifier_internal_call(kwargs): + return + if deployment_accepts_param(deployment, model, "reasoning_effort", host): + return + kwargs.pop("reasoning_effort", None) + proxy_server_request: Final = kwargs.get("proxy_server_request") + if not isinstance(proxy_server_request, dict): + return + body: Final[object] = proxy_server_request.get("body") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the proxy's untyped request + if isinstance(body, dict): + body.pop("reasoning_effort", None) # pyright: ignore[reportUnknownMemberType] # same + + +# --- timeouts --------------------------------------------------------------------------------- + + +def stream_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object: + """``_get_stream_timeout``.""" + return ( + kwargs.get("stream_timeout", None) + or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier") + or data.get("stream_timeout", None) + or shaping.stream_timeout + or shaping.request_timeout + or shaping.default_litellm_params.get("stream_timeout", None) + ) + + +def non_stream_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object: + """``_get_non_stream_timeout``.""" + return ( + kwargs.get("timeout", None) + or kwargs.get("request_timeout", None) + or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier") + or data.get("timeout", None) + or data.get("request_timeout", None) + or shaping.request_timeout + or shaping.timeout + or shaping.default_litellm_params.get("timeout", None) + ) + + +def call_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object: + """``_get_timeout``.""" + stream: Final = stream_timeout(shaping, kwargs, data) if kwargs.get("stream") else None + return stream if stream is not None else non_stream_timeout(shaping, kwargs, data) + + +# --- default litellm params ------------------------------------------------------------------- + + +def update_kwargs_with_default_litellm_params( + shaping: Shaping, + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + metadata_variable_name: str, +) -> None: + """``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request; + the default metadata is merged over the request's.""" + defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)) + metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + for key, value in defaults.items(): + if value is None: + continue + kwargs.setdefault(key, value) + metadata_bucket(kwargs, metadata_variable_name).update(as_dict(metadata_defaults)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + + +# --- clientside credentials ------------------------------------------------------------------- + + +def _clientside_deployment( + deployment: Mapping[str, object], kwargs: Mapping[str, object], function_name: str | None +) -> tuple[Mapping[str, object], str, object]: + """``_handle_clientside_credential``: the request-scoped model_info, model and api_base.""" + model_info: Final = as_dict(deployment.get("model_info")) + litellm_params: Final = as_dict(deployment.get("litellm_params")) + dynamic_litellm_params: Final = get_dynamic_litellm_params( # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + litellm_params=litellm_params, request_kwargs=dict(kwargs) + ) + metadata_variable_name: Final = get_router_metadata_variable_name(function_name=function_name) + model_group: Final = as_dict(kwargs.get(metadata_variable_name)).get("model_group") + model_id: Final = generate_deployment_id(model_group, dynamic_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownArgumentType] # the group as the request carries it + original_model_id: Final = model_info.get("id") + model_info["id"] = model_id + model_info["original_model_id"] = original_model_id + built: Final = Deployment( + model_name=model_group, # pyright: ignore[reportArgumentType] # the group as the request carries it + litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params), + model_info=model_info, # pyright: ignore[reportArgumentType] # model_info as configured + ) + return built.model_info.model_dump(), built.litellm_params.model, built.litellm_params.api_base + + +# --- the deployment -------------------------------------------------------------------------- + + +def update_kwargs_with_deployment( + shaping: Shaping, + deployment: Mapping[str, object], + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + *, + cache: "DualCache | None", + function_name: str | None = None, + splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, +) -> None: + """``_update_kwargs_with_deployment``, in place on the attempt's kwargs.""" + deployment_params: Final = as_dict(deployment.get("litellm_params")) + for key in _forwarded_alias_marker_keys_the_deployment_sets( + deployment_params, kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) + ): + kwargs.pop(key, None) + kwargs.pop("daily_reset_time", None) + defaults: Final = shaping.default_litellm_params + merge_tools_from_deployment(deployment_params, kwargs, defaults) + + model_info: Mapping[str, object] = as_dict( # rebind-ok: rebound as the Python router rebinds it + deployment.get("model_info") + ) + deployment_model: object = deployment_params.get("model") # rebind-ok: same + deployment_api_base: object = deployment_params.get("api_base") # rebind-ok: same + deployment_model_name: Final = deployment.get("model_name") + from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils + + for key in _malformed_request_params_the_deployment_sets(deployment_params, defaults, kwargs): + kwargs.pop(key) + declared_provider: Final = deployment_params.get("custom_llm_provider") + declared_model: Final = deployment_params.get("model") + deployment_provider: Final = ( + declared_provider + if isinstance(declared_provider, str) + else inferred_provider(declared_model if isinstance(declared_model, str) else None) + ) + forwarded_authorization: Final = ( + ProviderSpecificHeaderUtils.forwarded_authorization(kwargs.get("provider_specific_header"), deployment_provider) + if deployment_provider in _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS + else None + ) + if is_clientside_credential(request_kwargs=dict(kwargs)) or forwarded_authorization is not None: + model_info, deployment_model, deployment_api_base = ( # rebind-ok: rebound as the Python router rebinds it + _clientside_deployment( + deployment, + {**kwargs, **({"api_key": forwarded_authorization} if forwarded_authorization is not None else {})}, + function_name, + ) + ) + + metadata_variable_name: Final = get_router_metadata_variable_name(function_name=function_name) + routing: Final = deployment.get(KUBERNETES_POD_ROUTING_KEY) + metadata: Final = metadata_bucket(kwargs, metadata_variable_name) + metadata.update( + { + "deployment": deployment_model, + "model_info": model_info, + "api_base": deployment_api_base, + "deployment_model_name": deployment_model_name, + KUBERNETES_POD_ROUTING_KEY: routing, + } + ) + other_bucket: Final = "metadata" if metadata_variable_name == "litellm_metadata" else "litellm_metadata" + other_bucket_metadata: Final = kwargs.get(other_bucket) + if isinstance(other_bucket_metadata, Mapping) and KUBERNETES_POD_ROUTING_KEY in other_bucket_metadata: + kwargs[other_bucket] = {**as_dict(other_bucket_metadata), KUBERNETES_POD_ROUTING_KEY: routing} # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + + if cache is not None: + refund_stale_reservation_before_retry(cache, kwargs) # pyright: ignore[reportArgumentType] # the request's kwargs dict + set_io_token_rate_limit_request_kwargs( + kwargs, # pyright: ignore[reportArgumentType] # the request's kwargs dict + store_in_context=deployment_has_io_token_limits(deployment), # pyright: ignore[reportArgumentType] # a deployment record + opens_responses_socket=opens_responses_socket, + metadata_channel=metadata_variable_name, + ) + metadata.setdefault( + ROUTING_REQUEST_TAGS_METADATA_KEY, + tuple(get_tags_from_request_kwargs(kwargs, metadata_variable_name=metadata_variable_name)), # pyright: ignore[reportArgumentType] # the request's kwargs dict + ) + deployment_tags: Final = deployment_params.get("tags") + merged_tags: Final = tags_with_serving_deployment_tags(metadata, deployment_tags) + if merged_tags is not None: + metadata["tags"] = list(merged_tags) + baggage_stamp: Final = baggage_stamp_for_serving_deployment(metadata, deployment_tags) + if baggage_stamp is not None: + metadata[BAGGAGE_MAPPED_TAGS_METADATA_KEY] = baggage_stamp + + credential_name: Final = deployment_params.get("litellm_credential_name") + if credential_name: + credential_tag: Final = f"Credential: {credential_name}" + existing_tags: Final = metadata.get("tags") or [] # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + if isinstance(existing_tags, list) and credential_tag not in existing_tags: + existing_tags.append(credential_tag) # pyright: ignore[reportUnknownMemberType] # the request's own tags list + metadata["tags"] = existing_tags + + from litellm.llms.github_copilot.per_user_auth import github_copilot_per_user_credential_name + + configured_per_user_name: Final = github_copilot_per_user_credential_name(deployment_params) + caller_credential_name: Final = kwargs.get("litellm_credential_name") + if ( + configured_per_user_name is not None + and isinstance(caller_credential_name, str) + and caller_credential_name != configured_per_user_name + ): + raise litellm_exceptions.BadRequestError( + message="litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth", + model=str(deployment_model_name), + llm_provider="", + ) + + kwargs["model_info"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + model_info + ) + _set_timeouts(shaping, deployment_params, kwargs, function_name, splits_stream_timeout) + + if router_config_requests_stream_usage(kwargs, (deployment_params, defaults)): + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + STREAM_USAGE_INJECTED_FIELD + ] = True + update_kwargs_with_default_litellm_params(shaping, kwargs, metadata_variable_name) + pinned_model_id: Final = deployment_model_id_pin(deployment_params) + if pinned_model_id is not None: + kwargs["model_id"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + pinned_model_id + ) + + +def _set_timeouts( + shaping: Shaping, + deployment_params: Mapping[str, object], + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + function_name: str | None, + splits_stream_timeout: bool, +) -> None: + if function_name == "_ageneric_api_call_with_fallbacks": + from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout + + router_timeout: Final = ( + shaping.request_timeout + if shaping.request_timeout is not None + else float(shaping.explicit_timeout) + if isinstance(shaping.explicit_timeout, (int, float)) + else None + ) + router_stream_timeout: Final = ( + shaping.stream_timeout + if shaping.stream_timeout is not None + else shaping.request_timeout + if shaping.request_timeout is not None + else shaping.default_litellm_params.get("stream_timeout") + ) + passthrough_timeout: Final = partial( + resolve_llm_passthrough_timeout, + litellm_params=deployment_params, + router_timeout=router_timeout, + router_stream_timeout=router_stream_timeout, # pyright: ignore[reportArgumentType] # a configured default, passed as the Python router passes it + ) + timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else dict(kwargs) + kwargs.update( + CompletionTimeout.stream_request_kwargs( + passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request) + ) + if splits_stream_timeout and timeout_request.get("stream") + else {"timeout": passthrough_timeout(kwargs=timeout_request)} + ) + elif splits_stream_timeout: + kwargs.update( + CompletionTimeout.stream_request_kwargs( + non_stream_timeout(shaping, kwargs, deployment_params), + stream_timeout(shaping, kwargs, deployment_params) if kwargs.get("stream") else None, + ) + ) + else: + kwargs["timeout"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + call_timeout(shaping, kwargs, deployment_params) + ) + + +def stamp_deployment_num_retries(error: BaseException, deployment: Mapping[str, object]) -> None: + """``_set_deployment_num_retries_on_exception``.""" + if getattr(error, "num_retries", None) is not None: + return + value: Final = as_dict(deployment.get("litellm_params")).get("num_retries") + if value is None: + return + try: + error.num_retries = int(str(value)) if not isinstance(value, int) else value # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + except (ValueError, TypeError): + pass + + +def stamp_failed_deployment_id(error: BaseException, model_info: object) -> None: + """``_set_failed_deployment_id_on_exception`` with the effective model_info.""" + if getattr(error, "failed_deployment_id", None): + return + deployment_id: Final = as_dict(model_info).get("id") + if deployment_id: + try: + error.failed_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + except Exception: # noqa: BLE001, S110 # as the Python router: a read-only error keeps its attributes + pass + + +def stamp_retry_skip_deployment_id(error: BaseException, kwargs: Mapping[str, object]) -> None: + """``_stamp_retry_skip_deployment_id``.""" + model_info: Final = kwargs.get("model_info") + if not isinstance(model_info, Mapping): + return + info: Final = as_dict(model_info) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + configured: Final = ( + info.get("original_model_id") if isinstance(error, litellm_exceptions.UnsupportedEndpointError) else None + ) + deployment_id: Final = configured or info.get("id") + if isinstance(deployment_id, str) and deployment_id: + error.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic stamp, read on retry # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + + +NO_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) + +__all__ = [ + "STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS", + "Shaping", + "as_dict", + "call_timeout", + "deployment_params_with_request_reasoning_override", + "drop_unsupported_classifier_reasoning_effort", + "merge_tools_from_deployment", + "metadata_bucket", + "stamp_deployment_num_retries", + "stamp_failed_deployment_id", + "stamp_retry_skip_deployment_id", + "update_kwargs_with_default_litellm_params", + "update_kwargs_with_deployment", +] diff --git a/litellm/router_rust/invoke/slots.py b/litellm/router_rust/invoke/slots.py new file mode 100644 index 00000000000..08e88b5b2f1 --- /dev/null +++ b/litellm/router_rust/invoke/slots.py @@ -0,0 +1,107 @@ +""" +A deployment's ``max_parallel_requests`` slot on the Rust router core (``_deployment_slot`` and +``_get_client(client_type="max_parallel_requests")``): the attempt holds one of the deployment's +slots around its provider call, and an open chat stream keeps it until the stream is exhausted, +closed, or falls back. A caller arriving while every slot is taken gets a 429, which the core +records as that attempt's failure. + +The limit object is the one the Python router caches (``MaxParallelRequestsLimit``), kept in the +core's routing cache under the same key, so its in-flight count is shared by every attempt. +""" + +import contextlib +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +from litellm._internal_context import with_service_target +from litellm.router_rust.invoke.shaping import ROUTER_CLIENTS_TARGET, as_dict + +if TYPE_CHECKING: + from litellm.router_rust.invoke.endpoints import EndpointCore, Plan + +# The attempt helpers that run inside `_deployment_slot` (`_acompletion` holds the same slot +# itself). Sync helpers, rerank, moderation and the search helper hold none. +_SLOTTED_KINDS: Final = frozenset( + { + "chat", + "text", + "embedding", + "image", + "transcription", + "speech", + "adapter", + "generic", + "batch", + "cancel_batch", + "vector_store_create", + } +) + + +@with_service_target(ROUTER_CLIENTS_TARGET) +def max_parallel_requests_limit( + core: "EndpointCore", deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> object: + """`_get_client(client_type="max_parallel_requests")`: the cached limit, built on first use + (`InitalizeCachedClient.set_max_parallel_requests_client`).""" + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + from litellm.utils import calculate_max_parallel_requests + + model_id: Final = as_dict(deployment.get("model_info")).get("id") + key: Final = f"{model_id}_max_parallel_requests_client" + span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) + cache: Final = core.routing_cache + cached: Final[object] = cache.get_cache(key=key, local_only=True, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # the cache holds untyped values + if cached is not None: + return cached # pyright: ignore[reportUnknownVariableType] # the cache holds untyped values + params: Final = as_dict(deployment.get("litellm_params")) + calculated: Final = calculate_max_parallel_requests( + rpm=params.get("rpm"), # pyright: ignore[reportArgumentType] # the configured value, as the Python router reads it + max_parallel_requests=params.get("max_parallel_requests"), # pyright: ignore[reportArgumentType] # same + tpm=params.get("tpm"), # pyright: ignore[reportArgumentType] # same + default_max_parallel_requests=core.endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given, as the Python router passes it + ) + if calculated: + cache.set_cache( # pyright: ignore[reportUnknownMemberType] # the routing cache + key=key, + value=MaxParallelRequestsLimit( + max_parallel_requests=calculated, + model_id=str(model_id), + model_group=str(deployment.get("model_name", "")), + ), + local_only=True, + ) + return cache.get_cache(key=key, local_only=True, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # see above + + +def enter_deployment_slot( + core: "EndpointCore", plan: "Plan", deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> contextlib.ExitStack: + """The attempt's slot, entered (a full deployment raises its 429 here); an empty stack when + the plan's helper holds none or the deployment has no limit.""" + from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + + slot: Final = contextlib.ExitStack() + if plan.sync or plan.kind not in _SLOTTED_KINDS: + return slot + limit: Final = max_parallel_requests_limit(core, deployment, kwargs) + if isinstance(limit, MaxParallelRequestsLimit): + slot.enter_context(limit) + return slot + + +def deployment_slot( + core: "EndpointCore", deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> contextlib.ExitStack: + """`_deployment_slot` outside a plan's attempt (a file created on every deployment).""" + from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + + slot: Final = contextlib.ExitStack() + limit: Final = max_parallel_requests_limit(core, deployment, kwargs) + if isinstance(limit, MaxParallelRequestsLimit): + slot.enter_context(limit) + return slot + + +__all__ = ["deployment_slot", "enter_deployment_slot", "max_parallel_requests_limit"] diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py new file mode 100644 index 00000000000..ca70143072f --- /dev/null +++ b/litellm/router_rust/invoke/streaming.py @@ -0,0 +1,1839 @@ +""" +The stream wrappers of the Rust router core: host-side objects around a streamed response. + +Every attempt the Rust core runs that returns a stream is wrapped here before the core sees the +outcome, the way the Python router wraps each attempt's stream: + +- a chat stream (``CustomStreamWrapper``) has its first chunk read inside the attempt, so a stream + that fails before any token fails the attempt and the core retries or falls back as for any error +- a chat, ``/v1/messages`` or Responses stream that fails later, before the caller received content, + hands the failure back to the core: the core decides whether a same-group retry or a fallback takes + over (``RouterCoreHandle.stream_policy`` / ``retry_stream`` / ``fallback_stream``), and the wrapper + splices what that call returns into the stream the caller already holds + +Each wrapper also reports its stream's outcome to the core when the stream ends, as litellm's stream +logging reports it to the Python router's callbacks: the attempt that returned the stream counted +only its request (the attempt answer is ``deferred``), so the success with the stream's final tokens, +or a failure the stream raised after it opened, is recorded then (``StreamAttempt.report_success`` / +``report_failure``). A stream closed before it ends reports nothing. + +The wrappers decide nothing about routing: what may take over, the retry budget and every +deployment pick come from the core. They hold live Python iterators, frames and hidden params, +which is why they live in the host. Ported from the Python router's ``_acompletion_streaming_iterator``, +``_completion_streaming_iterator``, ``_aanthropic_messages_streaming_iterator`` and +``_aresponses_streaming_iterator``, without importing it. +""" + +import asyncio +import contextlib +import dataclasses +import json +import weakref +from collections.abc import ( + AsyncGenerator, + AsyncIterable, + AsyncIterator, + Awaitable, + Callable, + Coroutine, + Generator, + Iterable, + Iterator, + Mapping, + Sequence, +) +from typing import ( + TYPE_CHECKING, + Final, + Generic, + NoReturn, + TypeAlias, + TypeVar, + cast, # noqa: TID251 # untyped stream iterators and provider chunk lists +) + +import anyio + +import litellm.exceptions as litellm_exceptions +from litellm._logging import verbose_router_logger +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + add_retry_headers_to_response, + get_hidden_params_dict, + replace_complexity_router_headers, +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, without +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.router_rust.surface.health import failure_already_logged, logged_call +from litellm.types.utils import ModelResponseStream, Usage + +if TYPE_CHECKING: + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + from litellm.router_python_host import PythonHost + from litellm.router_rust.core import RustCore + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ContinuationOutput, ContinuedPartial + from litellm.types.llms.openai import ResponseAPIUsage, ResponseIncompleteEvent + +# A stream that never emits content or an error could grow a held-frame buffer without bound, so +# hitting either cap commits the stream instead. +MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS: Final = 200 +MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: Final = 200 + +# The hidden params and headers a fallback's response carries, as dicts the proxy writes into. +_Prepared: TypeAlias = "tuple[dict[str, object], dict[str, object]]" +_RestT: Final = TypeVar("_RestT", Iterator[object], AsyncIterator[object]) + + +# --- the attempt a stream came from ------------------------------------------------------------- + + +@dataclasses.dataclass(slots=True) +class StreamAttempt: + """The attempt that returned a stream, and the core that can still recover it. + + ``request`` is the call's prepared request, the one every attempt of the call is built from; + ``kwargs`` are this attempt's own kwargs. The origin fields are what the core reads to decide + what may take over a failure of this stream. + """ + + core: "RustCore" + op: str + request: Mapping[str, object] + kwargs: Mapping[str, object] + group: str + order: int | None + depth: int + level: int + deployment_id: str | None + attempted_retries: int + committed_budget: int | None = None + # The deployment's max_parallel_requests slot the attempt holds; an open chat stream takes it. + slot: contextlib.ExitStack | None = None + # Set by the wrapper that reports the stream's outcome when it ends; the attempt's answer is then + # deferred and counts only the request and the tokens read so far (``counted_tokens``). + reports_outcome: bool = False + counted_tokens: int = 0 + outcome_reported: bool = False + # The background recording of the stream's failure, which a same-group retry waits for so it + # picks with the failed deployment already cooled down. + failure_recording: "asyncio.Task[None] | None" = None + + @property + def model_group(self) -> str: + return self.group + + def origin(self) -> Mapping[str, object]: + return { + "group": self.group, + "order": self.order, + "depth": self.depth, + "level": self.level, + "deployment_id": self.deployment_id, + "attempted_retries": self.attempted_retries, + "committed_budget": self.committed_budget, + } + + def policy(self, trigger: BaseException | None = None) -> Mapping[str, object]: + return self.core.stream_policy(self.request, self.origin(), trigger) + + def can_retry(self) -> bool: + return self.policy().get("can_retry") is True + + def can_fall_back(self) -> bool: + return self.policy().get("can_fall_back") is True + + def refusal_fallback_available(self) -> bool: + return self.policy().get("refusal_fallback") is True + + def retry_budget(self, trigger: BaseException) -> tuple[int, bool]: + policy: Final = self.policy(trigger) + budget: Final = policy.get("retry_budget") + return (budget if isinstance(budget, int) else 0), policy.get("policy_applies") is True + + def should_retry_status(self, status_code: int) -> bool: + return bool(self.core._host.invoke("should_retry")(status_code)) # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the wrapper serves its own core's host + + async def fallback(self, trigger: BaseException, request: Mapping[str, object] | None = None) -> object: + """The fallback walk for this stream's failure; raises when no fallback answers.""" + return await self.core.recover_stream("fallback", self.op, request or self.request, self.origin(), trigger) + + def fallback_sync(self, trigger: BaseException) -> object: + return self.core.recover_stream_sync("fallback", self.op, self.request, self.origin(), trigger) + + async def retry(self, trigger: BaseException) -> "tuple[object, int, int] | BaseException": + """A same-group retry: the retried response with the retry count and budget, or the newest failure.""" + return await self.core.retry_stream(self.op, self.request, self.origin(), trigger) + + # -- the stream's outcome, reported once when it ends -------------------------------------- + + def logged(self) -> dict[str, object]: # mutable-ok: litellm's logging kwargs + """The logging kwargs litellm's stream logging hands the outcome callbacks, for this attempt.""" + from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY + + kwargs: Final = self.kwargs + metadata: Final = as_str_mapping(kwargs.get("litellm_metadata")) or as_str_mapping(kwargs.get("metadata")) or {} + return { + "call_type": self.op, + "litellm_params": { + "metadata": {**metadata, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: self.counted_tokens}, + "model_info": kwargs.get("model_info") or {}, + "api_base": kwargs.get("api_base"), + "client_side_timeout": kwargs.get("client_side_timeout"), + "timeout": kwargs.get("timeout"), + **({"cooldown_time": kwargs["cooldown_time"]} if "cooldown_time" in kwargs else {}), + }, + } + + def _failure_logged(self, error: BaseException) -> dict[str, object]: # mutable-ok: litellm's logging kwargs + return {**logged_call(self.op, self.kwargs), **self.logged(), "exception": _provider_error(error)} + + def _callbacks(self, *, sync: bool) -> Mapping[str, bool]: + """The outcome callbacks that run for this attempt when its stream ends.""" + return self.core.outcome_callbacks(self.kwargs, sync=sync) + + def _claim(self) -> bool: + if not self.reports_outcome or self.outcome_reported: + return False + self.outcome_reported = True + return True + + def report_success(self, total_tokens: int, last: object = None) -> None: + """The stream ended: count its success and the tokens the attempt did not count, and re-pin + the session on ``last``, the stream's final item (``deployment_callback_on_success``), in the + background.""" + if self._claim(): + logged: Final = {**self.logged(), "standard_logging_object": {"total_tokens": total_tokens}} + _record_in_background(self.core.report_stream_success(logged, last, self._callbacks(sync=False))) + + def report_failure(self, error: BaseException) -> None: + """The stream failed after it opened: count the failure, cool down by the rules and count one + more request (``async_deployment_callback_on_failure``), in the background. A logging object + shared with an earlier failure blocks it: the stream logs through ``attempt_scoped_copy``, read + here, when the stream fails.""" + if self._claim() and not failure_already_logged(self.kwargs, "async_failure"): + self.failure_recording = _record_in_background( + self.core.report_stream_failure(self._failure_logged(error), self._callbacks(sync=False)) + ) + + async def failure_recorded(self) -> None: + """Wait for the stream's failure to be recorded, if one is being recorded.""" + recording: Final = self.failure_recording + if recording is not None and not recording.done(): + _ = await asyncio.wait({recording}) + + def report_first_chunk_failure(self, error: BaseException) -> None: + """The stream failed before its attempt returned it. ``_dispatch_stream_failure`` logs that in + a task of its own, so the record reads the error as the router left it by then, with its + routing note when no real suspension came first. The attempt checked the logging object.""" + _record_in_background(self.core.report_stream_failure(self._failure_logged(error), self._callbacks(sync=False))) + + def report_success_sync(self) -> None: + """A sync stream ended: a sync call's success callback counts the success alone.""" + if self._claim(): + _record_now(lambda: self.core.report_stream_success_sync(self.logged(), self._callbacks(sync=True))) + + def report_failure_sync(self, error: BaseException) -> None: + """A sync stream failed after it opened: counted and cooled down, with no request counted.""" + if self._claim() and not failure_already_logged(self.kwargs, "sync_failure"): + logged: Final = self._failure_logged(error) + _record_now(lambda: self.core.report_stream_failure_sync(logged, self._callbacks(sync=True))) + + +# --- reporting a stream's outcome --------------------------------------------------------------- + +# The background recordings still running, held so none is collected before it finishes. +_RECORDINGS: Final[set["asyncio.Task[None]"]] = set() # mutable-ok: tasks leave it when they finish + + +def _provider_error(error: BaseException) -> BaseException: + """The provider's own error, which the failure rules classify, rather than the router's envelope.""" + if isinstance(error, MidStreamFallbackError) and error.original_exception is not None: + return error.original_exception + return error + + +def _record_in_background(recording: Coroutine[object, object, object]) -> "asyncio.Task[None]": + """Run a recording through the core's async cache driver without holding up the stream, as litellm + schedules its stream logging.""" + + async def run() -> None: + try: + _ = await recording + except Exception as error: # noqa: BLE001 # a recording never fails the stream + verbose_router_logger.warning("a stream's outcome was not recorded: %s", error) + + task: Final = asyncio.get_running_loop().create_task(run()) + _RECORDINGS.add(task) + task.add_done_callback(_RECORDINGS.discard) + return task + + +def _record_now(recording: Callable[[], object]) -> None: + try: + _ = recording() + except Exception as error: # noqa: BLE001 # see _record_in_background + verbose_router_logger.warning("a stream's outcome was not recorded: %s", error) + + +_ItemT: Final = TypeVar("_ItemT") + + +async def _areported( + attempt: StreamAttempt, + items: AsyncIterable[_ItemT], + total_tokens: Callable[[object], int], + observe: Callable[[object], None] | None = None, +) -> AsyncGenerator[_ItemT, None]: + """``items``, reporting the attempt's stream once they end: a success with the tokens + ``total_tokens`` reads off the last item, or the failure they raise. Closed early, nothing is + reported. The wrapper marks the attempt (``reports_outcome``) when it wraps the stream, before + the attempt answers.""" + last: object = None # rebind-ok: the newest item + try: + async for item in items: + last = item + if observe is not None: + observe(item) + yield item + except Exception as error: + attempt.report_failure(error) + raise + attempt.report_success(total_tokens(last), last) + + +def _reported(attempt: StreamAttempt, items: Iterable[_ItemT]) -> Generator[_ItemT, None, None]: + """``_areported`` for a sync stream, whose success counts no tokens.""" + try: + yield from items + except Exception as error: + attempt.report_failure_sync(error) + raise + attempt.report_success_sync() + + +def _usage_total_tokens(usage: object) -> int: + mapping: Final = as_str_mapping(usage) + tokens: Final[object] = mapping.get("total_tokens") if mapping is not None else getattr(usage, "total_tokens", None) + return tokens if isinstance(tokens, int) and not isinstance(tokens, bool) and tokens > 0 else 0 + + +def _chat_total_tokens(last: object) -> int: + """The total tokens of a finished chat stream: its usage chunk's, else the final usage its last + chunk's hidden params carry once the stream assembled its response.""" + return _usage_total_tokens(getattr(last, "usage", None)) or _usage_total_tokens( + (get_hidden_params(last) or {}).get("usage") + ) + + +def _responses_total_tokens(last: object, source: object) -> int: + """The total tokens of a finished Responses stream: its terminal event's response usage.""" + for event in (last, getattr(source, "completed_response", None)): + tokens = _usage_total_tokens(getattr(getattr(event, "response", None), "usage", None)) + if tokens: + return tokens + return 0 + + +class _AnthropicStreamUsage: + """The usage a ``/v1/messages`` stream reports in its frames, as litellm logs it: the prompt's input, + cache-read and cache-write tokens on ``message_start`` (a ``message_delta`` may restate them), and the + cumulative output tokens on each ``message_delta``. A frame split across byte chunks is read once its + line completes.""" + + __slots__ = ("_open_line", "_output", "_prompt") + + def __init__(self) -> None: + self._open_line = b"" + self._prompt: dict[str, int] = {} # mutable-ok: the newest count per prompt field + self._output = 0 + + def observe(self, chunk: object) -> None: + if not isinstance(chunk, (bytes, bytearray)): + return + *lines, self._open_line = (self._open_line + bytes(chunk)).split(b"\n") + for line in lines: + if line.startswith(b"data:") and b'"usage"' in line: + self._read(line[len(b"data:") :]) + + def _read(self, payload: bytes) -> None: + try: + event: Final[object] = json.loads(payload) # pyright: ignore[reportAny] # plain JSON + except ValueError: + return + frame: Final = as_str_mapping(event) or {} + kind: Final = frame.get("type") + usage: Final = as_str_mapping( + (as_str_mapping(frame.get("message")) or {}).get("usage") if kind == "message_start" else frame.get("usage") + ) + if usage is None or kind not in ("message_start", "message_delta"): + return + for field in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"): + count = usage.get(field) + if isinstance(count, int) and not isinstance(count, bool): + self._prompt[field] = count + output: Final = usage.get("output_tokens") + if isinstance(output, int) and not isinstance(output, bool): + self._output = output + + def total_tokens(self, _last: object = None) -> int: + return sum(self._prompt.values()) + self._output + + +# --- the first chunk ---------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True, slots=True) +class _FirstChunk(Generic[_RestT]): + """The chunk an attempt already read off a stream (None: the provider closed it empty) and the + iterator it was read from, which carries on after it.""" + + chunk: object + rest: _RestT + + +def _mid_stream_retry_trigger(error: MidStreamFallbackError) -> Exception: + """The provider's own error, which is what the retry policy and the retry rules classify.""" + return error.original_exception if error.original_exception is not None else error + + +async def _aread_first_chunk(stream: CustomStreamWrapper) -> _FirstChunk[AsyncIterator[object]]: + """Read the first chunk while the attempt still holds the request, so a stream that fails before + any token fails the attempt. Iterates the way ``async for`` does: one ``__aiter__`` call, then + ``__anext__`` on what it returned.""" + rest: Final = stream.__aiter__() + try: + return _FirstChunk(chunk=await rest.__anext__(), rest=rest) + except StopAsyncIteration: + return _FirstChunk(chunk=None, rest=rest) + except MidStreamFallbackError as e: + raise _mid_stream_retry_trigger(e) from e + + +def _read_first_chunk(stream: CustomStreamWrapper) -> _FirstChunk[Iterator[object]]: + rest: Final = iter(stream) + try: + return _FirstChunk(chunk=next(rest), rest=rest) + except StopIteration: + return _FirstChunk(chunk=None, rest=rest) + except MidStreamFallbackError as e: + raise _mid_stream_retry_trigger(e) from e + + +def _chunks_with_first(model_response: Iterable[object], taken: _FirstChunk[Iterator[object]]) -> Iterator[object]: + if taken.chunk is None: + return + yield taken.chunk + yield from taken.rest + + +async def _achunks_with_first( + model_response: AsyncIterable[object], taken: _FirstChunk[AsyncIterator[object]] +) -> AsyncIterator[object]: + if taken.chunk is None: + return + yield taken.chunk + async for item in taken.rest: + yield item + + +# --- shared helpers ----------------------------------------------------------------------------- + + +_CONTENT_DELTA_KEYS: Final = ( + "content", + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", +) + + +def _delta_has_content(delta: object) -> bool: + get: Final[object] = getattr(delta, "get", None) + if not callable(get): + return False + for key in _CONTENT_DELTA_KEYS: + value: object = get(key) + if value: + return True + return False + + +def _stream_chunks_have_generated_content(chunks: Sequence[object]) -> bool: + for chunk in chunks: + choices: object = getattr(chunk, "choices", None) + if not isinstance(choices, Sequence) or not choices: + continue + delta: object = getattr(choices[0], "delta", None) # pyright: ignore[reportUnknownArgumentType] # a chunk choice + if _delta_has_content(delta): + return True + return False + + +def _raise_original_mid_stream_error(e: MidStreamFallbackError) -> NoReturn: + """Surface the provider error the stream wrapper carried instead of the internal MidStreamFallbackError.""" + if e.original_exception is not None: + raise e.original_exception from e + raise e + + +def _raise_fallback_failure(error: Exception) -> NoReturn: + """What a stream's recovery ends on: the error the fallback's own stream raised, or the failure + a walk that served nothing was handed; never the internal MidStreamFallbackError.""" + if isinstance(error, MidStreamFallbackError) and error.original_exception is not None: + raise error.original_exception from error + raise error + + +def _stream_pre_content_error(error: Exception, model: str, llm_provider: str = "anthropic") -> MidStreamFallbackError: + """The envelope the fallback chain judges a failure by when the client has received no content yet.""" + return MidStreamFallbackError( + message=str(error), + model=model, + llm_provider=llm_provider, + original_exception=error, + is_pre_first_chunk=True, + ) + + +def _context_window_stream_fallback_error( + error: "MidStreamFallbackError | litellm_exceptions.ContextWindowExceededError", + model: object, + llm_provider: str, + has_output: bool, +) -> MidStreamFallbackError: + if isinstance(error, litellm_exceptions.ContextWindowExceededError): + if has_output: + raise error + return _stream_pre_content_error(error, model if isinstance(model, str) else "", llm_provider) + if isinstance(error.original_exception, litellm_exceptions.ContextWindowExceededError) and ( + has_output or error.generated_content or error.emitted_disqualifying_content + ): + raise error.original_exception + return error + + +def _is_context_window(error: "MidStreamFallbackError | litellm_exceptions.ContextWindowExceededError") -> bool: + return isinstance(error, litellm_exceptions.ContextWindowExceededError) or isinstance( + error.original_exception, litellm_exceptions.ContextWindowExceededError + ) + + +def _fallback_trigger(e: MidStreamFallbackError, unwrapped: tuple[type[Exception], ...]) -> Exception: + """The error the fallback walk dispatches on: the provider's own for the kinds with a list of their own.""" + return e.original_exception if isinstance(e.original_exception, unwrapped) else e + + +def _prepare_fallback_hidden_params( + fallback_response: object, +) -> _Prepared: + fallback_hidden_params: Final = get_hidden_params_dict(fallback_response) + fallback_headers: Final = fallback_hidden_params.get("additional_headers") + if not isinstance(fallback_headers, dict): + return fallback_hidden_params, {} + return fallback_hidden_params, cast("dict[str, object]", fallback_headers) # cast-ok: a dict of headers + + +def _apply_fallback_hidden_params_to_item( + fallback_item: object, + prepared_fallback_hidden_params: _Prepared, +) -> None: + if fallback_item is None or not hasattr(fallback_item, HIDDEN_PARAMS_ATTR): + return + fallback_hidden_params, fallback_headers = prepared_fallback_hidden_params + item_hidden_params: Final = get_hidden_params_dict(fallback_item) + item_headers: object = item_hidden_params.get("additional_headers") # rebind-ok: replaced when not a dict + if not isinstance(item_headers, dict): + item_headers = {} # rebind-ok: rebound as the stream advances + set_hidden_params( + fallback_item, + { + **item_hidden_params, + **fallback_hidden_params, + "additional_headers": {**cast("dict[str, object]", item_headers), **fallback_headers}, # cast-ok: checked + }, + ) + + +def _combine_fallback_usage(fallback_item: ModelResponseStream, complete_response_object_usage: Usage | None) -> None: + """Merge partial-stream usage with fallback-stream usage on the chunk. A usage given as a plain + dict comes out all-zero: ``combine_usage_objects`` reads token counts as attributes.""" + from litellm.cost_calculator import BaseTokenUsageProcessor + + usage: Final = cast("Usage | None", getattr(fallback_item, "usage", None)) # cast-ok: a chunk's usage + usage_objects: Final = [usage] if usage is not None else [] + if complete_response_object_usage is not None: + usage_objects.append(complete_response_object_usage) + fallback_item.usage = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=usage_objects) # pyright: ignore[reportAttributeAccessIssue] # usage is not a declared chunk field # rebind-ok: the Python router rewrites the chunk's usage in place + + +# --- mid-stream continuation -------------------------------------------------------------------- + +CONTINUATION_FLAG: Final = "enable_mid_stream_fallback_continuation" +NO_FLAG: Final = object() +# The routing-body key naming the deployments a continuation may not land on (``continuation_deployments``). +CONTINUATION_EXCLUDED_IDS: Final = "continuation_excluded_ids" + + +def continuation_flag(value: object) -> bool: + """``_is_flag_enabled``: True, or a string ``str_to_bool`` reads as True.""" + from litellm.secret_managers.main import str_to_bool + + return value is True or (isinstance(value, str) and str_to_bool(value) is True) + + +def register_continuation_prefill_check(host: "PythonHost") -> None: + """``_register_continuation_prefill_check``: process-global on purpose, as in the Python router.""" + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ContinuationPrefillDeploymentCheck + + host.callbacks.add_litellm_callback(ContinuationPrefillDeploymentCheck()) + + +def continuation_routing(core: "RustCore", request: Mapping[str, object]) -> Mapping[str, object]: + """The routing fields of a continuation request: the deployments that cannot continue it, which the + core leaves out of selection the way ``continuation_deployments`` filters the healthy list.""" + from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_MARKER, + deployment_can_continue, + ) + + if request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER: + return {} + registry: Final = core.registry() + excluded: Final = [ + deployment.model_info.id + for record in registry.all() + if (deployment := registry.deployment(str(_record_id(record)))) is not None + and not deployment_can_continue(deployment.model_dump(exclude_none=True)) + ] + return {"body": {CONTINUATION_EXCLUDED_IDS: excluded}} + + +def _record_id(record: Mapping[str, object]) -> object: + model_info: Final = record.get("model_info") + return model_info.get("id") if isinstance(model_info, Mapping) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a plain record + + +def _names_deployment_that_cannot_continue(core: "RustCore", target: object) -> bool: + from litellm.router_utils.fallback_event_handlers import fallback_target_name + from litellm.router_utils.pre_call_checks.continuation_prefill_check import deployment_can_continue + + name: Final = fallback_target_name(target) + registry: Final = core.registry() + model_names: Final = {record.get("model_name") for record in registry.all()} + if name is None or name in model_names or not registry.has_deployment(name): + return False + deployment: Final = registry.deployment(name) + return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True)) + + +def _mid_stream_continuation_eligible( + attempt: StreamAttempt, e: MidStreamFallbackError, cut_deployment_id: object +) -> bool: + """``_mid_stream_continuation_eligible``: whether a stream that broke after plain assistant text may + continue via prefill.""" + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.router_utils.fallback_event_handlers import ( + continuation_fallback_targets, + fallback_lookup_groups, + fallback_target_overrides_request_params, + ) + from litellm.router_utils.pre_call_checks.continuation_prefill_check import constrains_continuation + from litellm.types.guardrails import GuardrailEventHooks + + core: Final = attempt.core + if not core.enable_mid_stream_fallback_continuation: + return False + if not e.generated_content or e.emitted_disqualifying_content: + return False + if isinstance(e.original_exception, litellm_exceptions.ContentPolicyViolationError): + return False + if isinstance(cut_deployment_id, str) and not core.registry().has_deployment(cut_deployment_id): + return False + if constrains_continuation(_params_sent_to_deployment(attempt, cut_deployment_id)): + return False + request: Final = attempt.request + requested_group: Final = request.get("model") + lookup_groups: Final = fallback_lookup_groups( + request, requested_group if isinstance(requested_group, str) else None + ) + targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them + if any(fallback_target_overrides_request_params(target) for target in targets): + return False + if any(_names_deployment_that_cannot_continue(core, target) for target in targets): + return False + return not any( + isinstance(callback, CustomGuardrail) + and callback._event_hook_is_event_type( # pyright: ignore[reportPrivateUsage] # the only check of a guardrail's hooks + GuardrailEventHooks.post_call + ) + for callback in core._host.callback_list("callbacks") # pyright: ignore[reportPrivateUsage, reportArgumentType] # the core's own host; read as getattr(litellm, name) + ) + + +def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]: + """Continue the last text block in place so the prefill reads as the text the caller already received.""" + last: Final = blocks[-1] if blocks else None + if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str): + return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}] + return [*blocks, {"type": "text", "text": text}] + + +def _build_completion_continuation_input( + messages: Sequence[Mapping[str, object]], generated_content: str +) -> Sequence[Mapping[str, object]] | None: + """The partial output as an assistant prefill, or a trailing ``prefix: True`` assistant turn extended in + place. None when the request ends in a plain assistant turn, in a prefill turn whose content this cannot + extend, or when the cut came mid-echo of the prefill.""" + from pydantic import TypeAdapter, ValidationError + + from litellm.router_utils.pre_call_checks.continuation_prefill_check import continued_text + + last: Final = messages[-1] if messages else None + if last is None or last.get("role") != "assistant": + return [*messages, {"role": "assistant", "content": generated_content, "prefix": True}] + if last.get("prefix") is not True: + return None + new_text: Final = continued_text(last, generated_content) + if new_text is None: + return None + content: Final = last.get("content") + if content is None or isinstance(content, str): + return [*messages[:-1], {**last, "content": f"{content or ''}{new_text}", "prefix": True}] + try: + blocks: Final = TypeAdapter(list[dict[str, object]]).validate_python(content) + except ValidationError: + return None + return [*messages[:-1], {**last, "content": _extend_trailing_text_block(blocks, new_text), "prefix": True}] + + +def _price_cut_partial(cut_stream: CustomStreamWrapper, partial_response: object) -> "ContinuedPartial | None": + """Price the cut stream's partial, plus any partial it inherited, before a fallback can reuse its logging.""" + from litellm.cost_calculator import BaseTokenUsageProcessor + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ContinuedPartial + + cut_logging: Final = cut_stream.logging_obj + partial_usage: Final = getattr(partial_response, "usage", None) + if partial_response is None or not isinstance(partial_usage, Usage): + return None + details: Final = cut_logging.model_call_details + inherited_usage: Final = details.get("continued_partial_usage") + inherited_cost: Final = details.get("continued_partial_cost") + inherited_id: Final = details.get("continued_response_id") + priced_partial: Final = partial_response.model_copy( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # a ModelResponse + update={"model": cut_stream.model} if cut_stream.model else {} # pyright: ignore[reportUnknownMemberType] # untyped attribute + ) + priced_partial._hidden_params = without( # pyright: ignore[reportUnknownMemberType] # no public accessor + get_hidden_params_dict(partial_response), "response_cost" + ) + return ContinuedPartial( + usage=BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[partial_usage, inherited_usage] if isinstance(inherited_usage, Usage) else [partial_usage] + ), + response_cost=(cut_logging.response_cost_calculator(result=priced_partial) or 0.0) # pyright: ignore[reportUnknownArgumentType] # a ModelResponse + + (inherited_cost if isinstance(inherited_cost, float) else 0.0), + response_id=inherited_id if isinstance(inherited_id, str) else str(getattr(partial_response, "id", "")), + ) + + +def _carry_partial_spend_to_fallback(fallback_response: object, cut_partial: "ContinuedPartial | None") -> None: + """Bill what the cut legs consumed on the continuation's success log.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + fallback_logging: Final = getattr(fallback_response, "logging_obj", None) + if cut_partial is None or not isinstance(fallback_logging, LiteLLMLogging): + return + fallback_logging.record_continued_partial( + usage=cut_partial.usage, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # a Usage + response_cost=cut_partial.response_cost, + response_id=cut_partial.response_id, + ) + + +@dataclasses.dataclass(frozen=True, slots=True) +class _Continuation: + """What a chat continuation sends the fallback and splices back.""" + + request: Mapping[str, object] + output: "ContinuationOutput" + partial: "ContinuedPartial | None" + usage: Usage | None + + +async def _chat_continuation( + attempt: StreamAttempt, e: MidStreamFallbackError, model_response: CustomStreamWrapper +) -> _Continuation: + """The request a fallback continues a chat stream from, once it broke after content. Raises the + provider's error when the continuation is declined or the output ceiling is spent.""" + from litellm.constants import MID_STREAM_FAILURE_LOG_WAIT_SECONDS + from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG + from litellm.main import ( + stream_chunk_builder, # pyright: ignore[reportUnknownVariableType] # untyped chunk list argument + ) + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_MARKER, + ContinuationOutput, + continuation_output_ceilings, + ) + + request: Final = attempt.request + raw_messages: Final = request.get("messages") + messages: Final = cast("list[Mapping[str, object]]", raw_messages or []) # cast-ok: chat messages + complete: Final = stream_chunk_builder(chunks=model_response.chunks, messages=messages) # pyright: ignore[reportUnknownArgumentType] # provider chunk list + usage: Final = cast("Usage | None", getattr(complete, "usage", None)) # cast-ok: a built response's usage + continuation_messages: Final = _build_completion_continuation_input(messages, e.generated_content) + output: Final = ( + ContinuationOutput( + e.generated_content, + continuation_messages[-1], + str(request.get("model") or ""), + complete, + checks_post_call_rules=request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER, + ) + if continuation_messages is not None + else None + ) + partial: Final = _price_cut_partial(model_response, complete) if output is not None else None + if partial is not None: + model_response.logging_obj.mark_stash_billed_by_continuation() + emitted_tokens: Final = int(getattr(usage, "completion_tokens", 0) or 0) + reduced: Final = continuation_output_ceilings(request, emitted_tokens) + if reduced is None or continuation_messages is None or output is None: + _raise_original_mid_stream_error(e) + await model_response.wait_for_failure_logging(timeout=MID_STREAM_FAILURE_LOG_WAIT_SECONDS) + return _Continuation( + request={ + **request, + **reduced, + "messages": continuation_messages, + MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER, + }, + output=output, + partial=partial, + usage=usage, + ) + + +# --- chat completions --------------------------------------------------------------------------- + + +class FallbackAwareStreamWrapper(CustomStreamWrapper): + """A chat stream built around the attempt the core picked first, which repoints itself when a + fallback takes over.""" + + fallback_headers_adopted: bool = False + + def adopt_fallback_response_headers( + self, + fallback_response: object, + prepared_fallback_hidden_params: _Prepared, + ) -> None: + """Repoint this wrapper at the deployment that served the stream. Replaces rather than merges, + so the failed attempt's request id, counters, ``model_id`` and ``api_base`` cannot leak.""" + self._response_headers = getattr(fallback_response, "_response_headers", None) + fallback_hidden_params, fallback_headers = prepared_fallback_hidden_params + if fallback_hidden_params: + self._hidden_params = {**fallback_hidden_params, "additional_headers": dict(fallback_headers)} + self._base_hidden_params = {**self._hidden_params, "response_cost": None} + self.fallback_headers_adopted = True + + +def _adopt_fallback_response_headers( + wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object +) -> _Prepared: + prepared: Final = _prepare_fallback_hidden_params(fallback_response) + adopting_wrapper: Final = wrapper_ref() + if adopting_wrapper is not None: + adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared) + return prepared + + +def _chat_wrapper_init( + wrapper: FallbackAwareStreamWrapper, stream: object, model_response: CustomStreamWrapper +) -> None: + CustomStreamWrapper.__init__( # pyright: ignore[reportUnknownMemberType] # the base constructor is untyped + wrapper, + completion_stream=stream, + model=model_response.model, # pyright: ignore[reportUnknownMemberType] # an untyped attribute + custom_llm_provider=model_response.custom_llm_provider, + logging_obj=model_response.logging_obj, + _response_headers=getattr(model_response, "_response_headers", None), + ) + inner_chunks: Final[object] = getattr(model_response, "chunks", None) + if isinstance(inner_chunks, list): + chunk_list: Final = cast("list[ModelResponseStream]", inner_chunks) # cast-ok: the inner stream's chunks + wrapper.chunks = chunk_list # rebind-ok: initializes the wrapper being built + hidden_params: Final = get_hidden_params(model_response) + if hidden_params is not None: + wrapper._hidden_params = dict(hidden_params) # pyright: ignore[reportPrivateUsage] # the wrapper's own field # rebind-ok: initializes the wrapper being built + + +class _AsyncChatStream(FallbackAwareStreamWrapper): + def __init__(self, generator: AsyncGenerator[object, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it + _chat_wrapper_init(self, generator, model_response) + self._async_generator: Final = generator + + def __aiter__(self) -> "_AsyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator + return self + + async def __anext__(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # yields what the stream yields + return await self._async_generator.__anext__() + + +class _SyncChatStream(FallbackAwareStreamWrapper): + def __init__(self, generator: Generator[object, None, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it + _chat_wrapper_init(self, generator, model_response) + self.chunks = [] # the sync twin keeps no inner chunk list + hidden_params: Final = get_hidden_params(model_response) + if hidden_params is not None: + self._hidden_params = dict(hidden_params) + self._sync_generator: Final = generator + + def __iter__(self) -> "_SyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator + return self + + def __next__(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # yields what the stream yields + return next(self._sync_generator) + + +def _chat_has_output(model_response: CustomStreamWrapper) -> bool: + return bool(getattr(model_response, "response_uptil_now", "")) or _stream_chunks_have_generated_content( + model_response.chunks + ) + + +def wrap_chat_stream( # noqa: C901 # one function per Python wrapper it ports + attempt: StreamAttempt, + model_response: CustomStreamWrapper, + first_chunk: _FirstChunk[AsyncIterator[object]], + held_slot: contextlib.ExitStack | None = None, +) -> FallbackAwareStreamWrapper: + """``_acompletion_streaming_iterator``: a mid-stream error before content falls back through the core; + after content, the provider's error is raised. ``held_slot`` (the deployment's max_parallel_requests + slot) is released when the stream is exhausted, closed, or falls back.""" + slot: Final = held_slot if held_slot is not None else contextlib.ExitStack() + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + + async def close_model_response() -> None: + if not hasattr(model_response, "aclose"): + return + try: + await model_response.aclose() + except BaseException as e: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks: error closing model_response: %s", e) + + async def stream_with_fallbacks() -> AsyncGenerator[object, None]: + fallback_response: object = None # rebind-ok: tracked for cleanup + try: + async for item in _areported(attempt, _achunks_with_first(model_response, first_chunk), _chat_total_tokens): + yield item + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + getattr(model_response, "model", ""), + model_response.custom_llm_provider or "", + _chat_has_output(model_response), + ) + if _is_context_window(stream_error) + else cast("MidStreamFallbackError", stream_error) # cast-ok: not a context window error + ) + with anyio.CancelScope(shield=True): + await close_model_response() + slot.close() + committed: Final = bool( + not e.is_pre_first_chunk + and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) + ) + cut_deployment_id: Final = (get_hidden_params(model_response) or {}).get("model_id") + if committed and not _mid_stream_continuation_eligible(attempt, e, cut_deployment_id): + _raise_original_mid_stream_error(e) + trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,)) + try: + continuation: Final = await _chat_continuation(attempt, e, model_response) if committed else None + fallback_response = await attempt.fallback( # rebind-ok: rebound as the stream advances + trigger, continuation.request if continuation is not None else None + ) + if hasattr(fallback_response, "__aiter__"): + if continuation is not None: + _carry_partial_spend_to_fallback(fallback_response, continuation.partial) + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted + settled = False # rebind-ok: rebound as the stream advances + fallback_stream: Final = cast("AsyncIterable[object]", fallback_response) # cast-ok: checked + fallback_items: Final = ( + continuation.output.splice(fallback_stream) if continuation is not None else fallback_stream + ) + async for fallback_item in fallback_items: + if not settled: + settled = True + # a fallback that failed over again only repoints itself once it yields + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) + _apply_fallback_hidden_params_to_item(fallback_item, prepared) + if ( + fallback_item + and isinstance(fallback_item, ModelResponseStream) + and hasattr(fallback_item, "usage") + ): + _combine_fallback_usage( + fallback_item, + continuation.usage if continuation is not None else None, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # a Usage + ) + yield fallback_item + else: + # A fallback that answers without a stream yields None (an oddity kept as is). + yield None + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Fallback also failed: %s", fallback_error) + _raise_fallback_failure(fallback_error) + finally: + with anyio.CancelScope(shield=True): + await close_model_response() + slot.close() + if fallback_response is not None and hasattr(fallback_response, "aclose"): + try: + await fallback_response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except BaseException as close_error: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug( + "stream_with_fallbacks: error closing fallback_response: %s", close_error + ) + + wrapped_response: Final = _AsyncChatStream(stream_with_fallbacks(), model_response) + # weak, so the generator closing over it does not keep the wrapper alive + wrapper_ref: Final[weakref.ref[FallbackAwareStreamWrapper]] = weakref.ref(wrapped_response) + return wrapped_response + + +def wrap_chat_stream_sync( + attempt: StreamAttempt, model_response: CustomStreamWrapper, first_chunk: _FirstChunk[Iterator[object]] +) -> FallbackAwareStreamWrapper: + """``_completion_streaming_iterator``: the sync twin, which never continues after content.""" + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + + def stream_with_fallbacks() -> Generator[object, None, None]: + fallback_response: object = None # rebind-ok: tracked for cleanup + try: + yield from _reported(attempt, _chunks_with_first(model_response, first_chunk)) + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + getattr(model_response, "model", ""), + model_response.custom_llm_provider or "", + _chat_has_output(model_response), + ) + if _is_context_window(stream_error) + else cast("MidStreamFallbackError", stream_error) # cast-ok: not a context window error + ) + if attempt.request.get("disable_fallbacks") is True or ( + not e.is_pre_first_chunk + and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) + ): + _raise_original_mid_stream_error(e) + trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,)) + try: + fallback_response = attempt.fallback_sync(trigger) # rebind-ok: rebound as the stream advances + if hasattr(fallback_response, "__iter__"): + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted + settled = False # rebind-ok: rebound as the stream advances + for fallback_item in cast("Iterable[object]", fallback_response): # cast-ok: checked + if not settled: + settled = True + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) + _apply_fallback_hidden_params_to_item(fallback_item, prepared) + if ( + fallback_item + and isinstance(fallback_item, ModelResponseStream) + and hasattr(fallback_item, "usage") + ): + _combine_fallback_usage(fallback_item, None) + yield fallback_item + else: + yield None + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Fallback also failed: %s", fallback_error) + _raise_fallback_failure(fallback_error) + finally: + for closable in (model_response, fallback_response): + close = getattr(closable, "close", None) + if callable(close): + try: + close() + except BaseException as close_error: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks: error closing: %s", close_error) + + wrapped_response: Final = _SyncChatStream(stream_with_fallbacks(), model_response) + wrapper_ref: Final[weakref.ref[FallbackAwareStreamWrapper]] = weakref.ref(wrapped_response) + return wrapped_response + + +# --- /v1/messages ------------------------------------------------------------------------------- + + +class FallbackAwareAnthropicMessagesStream: + """A ``/v1/messages`` byte stream that carries the ``_hidden_params`` the proxy reads its response + headers off, through retries and fallbacks of the stream it wraps.""" + + def __init__( + self, async_generator: AsyncGenerator[bytes, None], source_iterator: object, attempt: StreamAttempt + ) -> None: + self._async_generator: Final = async_generator + self._source_iterator: object = source_iterator + self.attempt: Final = attempt + self.fallback_headers_adopted = False + self._hidden_params: dict[str, object] = dict( # mutable-ok: the proxy writes headers into it + getattr(source_iterator, "_hidden_params", None) or {} + ) + self._followed_source_params: object = None + + @property + def has_buffered_provider_output(self) -> bool: + return getattr(self._source_iterator, "has_buffered_provider_output", False) is True + + @property + def chunks(self) -> object: + return getattr(self._source_iterator, "chunks", None) + + @property + def messages(self) -> object: + return getattr(self._source_iterator, "messages", None) + + @property + def model(self) -> object: + return getattr(self._source_iterator, "model", None) + + def adopt_fallback_source(self, fallback_response: object) -> None: + self._source_iterator = fallback_response + self.fallback_headers_adopted = True + + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect + + return partial_responses_incomplete_event_on_disconnect(self._source_iterator) + + def follow_source_attribution(self) -> None: + """Mirror the source's current attribution onto this wrapper whenever the source has adopted a + new one: a retry's or a fallback's stream carries a wrapper of its own.""" + source: Final = self._source_iterator + if not getattr(source, "fallback_headers_adopted", False): + return + source_params: Final = getattr(source, "_hidden_params", None) + if source_params is None or source_params is self._followed_source_params: + return + self._followed_source_params = source_params + hidden_params, headers = _prepare_fallback_hidden_params(source) + self.merge_fallback_hidden_params(hidden_params, headers) + + def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": + return self + + async def __anext__(self) -> bytes: + return await self._async_generator.__anext__() + + async def aclose(self) -> None: + await self._async_generator.aclose() + + def merge_fallback_hidden_params( + self, fallback_hidden_params: Mapping[str, object], fallback_headers: Mapping[str, object] + ) -> None: + existing_headers: Final = cast( # cast-ok: additional_headers is a dict when present + "dict[str, object]", self._hidden_params.get("additional_headers") or {} + ) + self._hidden_params = { + **self._hidden_params, + **fallback_hidden_params, + "additional_headers": dict(replace_complexity_router_headers(existing_headers, fallback_headers)), + } + + +def _anthropic_stream_forwards_ping_live(chunk: object, has_generated_content: bool) -> bool: + from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_ping_chunk + + return not has_generated_content and is_anthropic_ping_chunk(chunk) + + +def _is_retriable_anthropic_status(status_code: int) -> bool: + return status_code == 429 or status_code >= 500 + + +def _anthropic_stream_error_is_gateway_verdict(chunk: object) -> bool: + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import is_server_fulfilled_tool_leak_error + + return is_server_fulfilled_tool_leak_error(chunk) + + +def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error: MidStreamFallbackError) -> bool: + return has_generated_content or not error.is_pre_first_chunk + + +def _stream_raised_error_status(error: Exception) -> int | None: + raw_status: Final = getattr(error, "status_code", None) + if isinstance(raw_status, int): + return raw_status + if isinstance(raw_status, str) and raw_status.isdigit(): + return int(raw_status) + response_status: Final = getattr(getattr(error, "response", None), "status_code", None) + return response_status if isinstance(response_status, int) else None + + +def _stream_fallback_error_for_raised( + attempt: StreamAttempt, error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" +) -> MidStreamFallbackError | None: + """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + if has_generated_content: + return None + status_code: Final = _stream_raised_error_status(error) + if status_code is None or attempt.should_retry_status(status_code): + return _stream_pre_content_error(error, model, llm_provider) + return None + + +def _stream_commits_now(chunk: object, has_generated_content: bool, buffered_chunk_count: int) -> bool: + from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_content_delta_chunk + + if has_generated_content: + return False + return is_anthropic_content_delta_chunk(chunk) or buffered_chunk_count >= MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS + + +def _bridged_stream_failure(source: object) -> Exception | None: + """The provider's error behind an ``event: error`` frame the completion or Responses bridge wrote + in its place, which litellm's stream logging logged. A native frame has none: litellm logs what the + frame itself maps to (``anthropic_error_frame_exception``), once a takeover closes the stream.""" + failure: Final[object] = getattr(source, "stream_failure", None) + return failure if isinstance(failure, Exception) else None + + +def _anthropic_recoverable_frame_error( + attempt: StreamAttempt, + error_event: tuple[str, str, int] | None, + chunk: object, + has_generated_content: bool, + model_group: str, +) -> Exception | None: + """The exception a provider ``event: error`` frame before content recovers through when a retry of + its class or a fallback can still take over; None forwards the frame as the provider sent it.""" + from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception + + if has_generated_content or error_event is None: + return None + error_type, message, status_code = error_event + if not _is_retriable_anthropic_status(status_code) or _anthropic_stream_error_is_gateway_verdict(chunk): + return None + frame_error: Final = anthropic_error_frame_exception(error_type, message, status_code, model_group) + budget, _ = attempt.retry_budget(frame_error) + if budget > attempt.attempted_retries: + return frame_error + if attempt.can_fall_back(): + return frame_error + return None + + +def wrap_anthropic_stream( + attempt: StreamAttempt, response: AsyncIterator[bytes] +) -> FallbackAwareAnthropicMessagesStream: + """``_aanthropic_messages_streaming_iterator``: lifecycle frames are held while a retry or a fallback + can still take over; an error frame, a raised error or a safeguard refusal before content is + retried in the group, then falls back, as the core decides.""" + from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + aclose_if_supported, + aclose_superseded_attempt, + parse_anthropic_error_event, + parse_anthropic_refusal_stop_details, + ) + from litellm.llms.anthropic.messages.native.utils import safeguard_refusal_error + + source_iterator: Final = response + replays_a_finished_agentic_turn: Final = isinstance(source_iterator, FakeAnthropicMessagesStreamIterator) + model: Final = attempt.group + usage: Final = _AnthropicStreamUsage() + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + + async def close_source() -> None: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_if_supported(source_iterator) + + async def close_superseded() -> None: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_superseded_attempt(source_iterator) + + async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: + policy: Final = attempt.policy() + has_generated_content = not ( # rebind-ok: set once content is seen, the buffer cap is hit, or nothing can take over + policy.get("can_retry") is True or policy.get("can_fall_back") is True + ) + buffered: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline + try: + async for chunk in _areported(attempt, source_iterator, usage.total_tokens, usage.observe): + if _anthropic_stream_forwards_ping_live(chunk, has_generated_content): + yield chunk + continue + if _stream_commits_now(chunk, has_generated_content, len(buffered)): + has_generated_content = True + # A transport can split one SSE data line across byte chunks, so pre-content detection + # parses the held frames plus the current chunk. + parse_window = ( + b"".join(c for c in (*buffered, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes + if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # see above + else chunk + ) + error_event = parse_anthropic_error_event(parse_window) + bridged_failure = _bridged_stream_failure(source_iterator) if error_event is not None else None + if bridged_failure is not None: + attempt.report_failure(bridged_failure) + recoverable_frame_error = _anthropic_recoverable_frame_error( + attempt, error_event, chunk, has_generated_content, model + ) + refusal_stop_details = ( + parse_anthropic_refusal_stop_details(parse_window) + if not has_generated_content and error_event is None + else None + ) + if ( + refusal_stop_details is not None + and not replays_a_finished_agentic_turn + and attempt.refusal_fallback_available() + ): + refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) + raise MidStreamFallbackError( + message=refusal_error.message, + model=model, + llm_provider="anthropic", + original_exception=refusal_error, + is_pre_first_chunk=True, + ) + if not has_generated_content and error_event is None: + buffered = (*buffered, chunk) + continue + if recoverable_frame_error is not None and error_event is not None: + raise MidStreamFallbackError( + message=error_event[1], + model=model, + llm_provider="anthropic", + original_exception=recoverable_frame_error, + is_pre_first_chunk=True, + ) + for buffered_chunk in buffered: + yield buffered_chunk + buffered = () + yield chunk + for buffered_chunk in buffered: + yield buffered_chunk + except Exception as stream_error: # noqa: BLE001 # any raised provider error must reach the fallback gate + async for item in _anthropic_recover_stream_error( + attempt, stream_error, has_generated_content, buffered, model, wrapper, close_superseded + ): + yield item + finally: + await close_source() + + wrapper: Final = FallbackAwareAnthropicMessagesStream(stream_with_fallbacks(), source_iterator, attempt) + return wrapper + + +async def _anthropic_recover_stream_error( + attempt: StreamAttempt, + stream_error: Exception, + has_generated_content: bool, + buffered: tuple[bytes, ...], + model: str, + wrapper: FallbackAwareAnthropicMessagesStream, + close_superseded: Callable[[], Awaitable[None]], +) -> AsyncGenerator[bytes, None]: + """Turns a source-iterator failure into a retry, a fallback, or the error reaching the caller.""" + if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, stream_error + ): + for buffered_chunk in buffered: + yield buffered_chunk + if stream_error.original_exception is not None: + raise stream_error.original_exception from stream_error + raise stream_error + fallback_error: Final = ( + stream_error + if isinstance(stream_error, MidStreamFallbackError) + else _stream_fallback_error_for_raised(attempt, stream_error, model, has_generated_content) + ) + if fallback_error is None: + raise stream_error + # A native error frame or a refusal the wrapper raised fails the attempt too (a raised error or a + # bridged frame already did). + attempt.report_failure(fallback_error) + await close_superseded() + await attempt.failure_recorded() + retried: Final = await attempt.retry(_mid_stream_retry_trigger(fallback_error)) + if isinstance(retried, BaseException): + last_error: Final = ( + fallback_error + if retried is _mid_stream_retry_trigger(fallback_error) + else _stream_pre_content_error( + retried if isinstance(retried, Exception) else fallback_error, model, fallback_error.llm_provider + ) + ) + async for item in _anthropic_fallback_attempt(attempt, last_error, wrapper): + yield item + return + response, attempted, budget = retried + inner: Final = response + if isinstance(inner, FallbackAwareAnthropicMessagesStream): + inner.attempt.attempted_retries = attempted + inner.attempt.committed_budget = budget + async for item in _anthropic_yield_recovered(inner, wrapper, (attempted, budget)): + yield item + + +async def _anthropic_yield_recovered( + recovered: object, + wrapper: FallbackAwareAnthropicMessagesStream, + retry_counters: tuple[int, int] | None = None, +) -> AsyncGenerator[bytes, None]: + """Hands a retry's or a fallback's response to the client through the wrapper, closing it after.""" + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + aclose_if_supported, + anthropic_messages_response_as_sse_events, + ) + + hidden_params, headers = _prepare_fallback_hidden_params(recovered) + wrapper.merge_fallback_hidden_params(hidden_params, headers) + wrapper.adopt_fallback_source(recovered) + if retry_counters is not None: + add_retry_headers_to_response( + response=wrapper, attempted_retries=retry_counters[0], max_retries=retry_counters[1] + ) + try: + if hasattr(recovered, "__aiter__"): + async for item in cast("AsyncIterator[bytes]", recovered): # cast-ok: __aiter__ checked above + wrapper.follow_source_attribution() + yield item + return + # A complete response for a streaming request is replayed as the SSE lifecycle a stream sends. + for event in anthropic_messages_response_as_sse_events(recovered): # pyright: ignore[reportArgumentType] # the non-streaming shape by elimination + yield event + finally: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_if_supported(recovered) + + +async def _anthropic_fallback_attempt( + attempt: StreamAttempt, e: MidStreamFallbackError, wrapper: FallbackAwareAnthropicMessagesStream +) -> AsyncGenerator[bytes, None]: + """Re-enters the fallback walk for a ``/v1/messages`` stream error and yields what the fallback produces.""" + trigger: Final = _fallback_trigger( + e, (litellm_exceptions.ContentPolicyViolationError, litellm_exceptions.ContextWindowExceededError) + ) + try: + fallback_response: Final = await attempt.fallback(trigger) + async for fallback_item in _anthropic_yield_recovered(fallback_response, wrapper): + yield fallback_item + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Anthropic messages streaming fallback also failed: %s", fallback_error) + _raise_fallback_failure(fallback_error) + + +# --- Responses API ------------------------------------------------------------------------------ + + +def _responses_stream_holds_event(item: object, held_event_count: int) -> bool: + from litellm.responses.streaming_iterator import PRE_OUTPUT_LIFECYCLE_EVENT_TYPES + + if held_event_count >= MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: + return False + return getattr(item, "type", None) in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES + + +def _responses_event_announces_background_run(item: object) -> bool: + return getattr(getattr(item, "response", None), "background", None) is True + + +def _mid_stream_error_before_output(error: MidStreamFallbackError) -> bool: + return not error.generated_content or error.is_pre_first_chunk + + +def _extract_partial_responses_usage(source_iterator: "BaseResponsesAPIStreamingIterator") -> "ResponseAPIUsage | None": + """Partial token usage of a Responses stream that failed, normalized to ``ResponseAPIUsage``.""" + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ) + + if isinstance(source_iterator, LiteLLMCompletionStreamingIterator): + chunks: Final = source_iterator.collected_chat_completion_chunks + if chunks: + try: + from litellm.main import ( + stream_chunk_builder, # pyright: ignore[reportUnknownVariableType] # untyped chunk list + ) + + built: Final = stream_chunk_builder(chunks=chunks) + chat: Final[object | None] = getattr(built, "usage", None) if built is not None else None + if chat is not None: + prompt: Final = int(getattr(chat, "prompt_tokens", 0) or 0) + completion: Final = int(getattr(chat, "completion_tokens", 0) or 0) + total: Final = int(getattr(chat, "total_tokens", prompt + completion) or (prompt + completion)) + return ResponseAPIUsage(input_tokens=prompt, output_tokens=completion, total_tokens=total) + except Exception as error: # noqa: BLE001 # the builder is best-effort + verbose_router_logger.debug("partial usage of a failed Responses stream: %s", error) + completed: Final = source_iterator.completed_response + if isinstance(completed, (ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent)): + return completed.response.usage + return None + + +def _combine_responses_fallback_usage(fallback_item: object, partial_usage: "ResponseAPIUsage") -> None: + from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ) + + if not isinstance(fallback_item, (ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent)): + return + response: Final = fallback_item.response + if response.usage is None: + return + fb: Final = response.usage + response.usage = ResponseAPIUsage( + input_tokens=(partial_usage.input_tokens or 0) + (fb.input_tokens or 0), + output_tokens=(partial_usage.output_tokens or 0) + (fb.output_tokens or 0), + total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0), + ) + + +def _params_sent_to_deployment(attempt: StreamAttempt, deployment_id: object) -> Mapping[str, object]: + """The request as the deployment received it, merged in the order the attempt applies.""" + return attempt.core.params_sent_to_deployment(attempt.request, deployment_id) + + +def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it ports + attempt: StreamAttempt, response: "BaseResponsesAPIStreamingIterator" +) -> "BaseResponsesAPIStreamingIterator": + """``_aresponses_streaming_iterator``: pre-output lifecycle events are held while a fallback can take + over; a mid-stream error falls back through the core, continuing after text.""" + import time + from datetime import datetime, timezone + + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, get_openai_response_types + + source_iterator: Final = response + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + openai_types: Final = get_openai_response_types() + terminal_event_types: Final = ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, + ) + + class FallbackResponsesStreamWrapper(BaseResponsesAPIStreamingIterator): + """Subclasses the base iterator only for isinstance checks; iteration is the generator's.""" + + fallback_headers_adopted: bool = False + + def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own + self._async_generator: Final = async_generator + self._billing_source: object = source_iterator + self._last_sequence_number = -1 + self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source + self.model = getattr(source_iterator, "model", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.logging_obj = getattr( + source_iterator, "logging_obj", getattr(source_iterator, "litellm_logging_obj", None) + ) # pyright: ignore[reportAttributeAccessIssue] # see above + self.finished = False + self.responses_api_provider_config = getattr(source_iterator, "responses_api_provider_config", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.completed_response = None + self.start_time = getattr(source_iterator, "start_time", None) or datetime.now(timezone.utc).replace( + tzinfo=None + ) + self._failure_handled = False + self._yielded_first_chunk = False + self._generated_content = "" + self._completed_response_cached = False + self._completed_response_logged = False + self._completed_response_cache_hit = None + self._persist_completed_response_before_logging = True + self._stream_created_time = time.time() + self.litellm_metadata = getattr(source_iterator, "litellm_metadata", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.custom_llm_provider = getattr(source_iterator, "custom_llm_provider", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.request_data = getattr(source_iterator, "request_data", {}) or {} + self.call_type = getattr(source_iterator, "call_type", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self._hidden_params = dict(getattr(source_iterator, "_hidden_params", None) or {}) + + def adopt_fallback_headers(self, fallback_response: object) -> _Prepared: + prepared: Final = _prepare_fallback_hidden_params(fallback_response) + self._billing_source = fallback_response + self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} + self.fallback_headers_adopted = True + return prepared + + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect + + current: Final = partial_responses_incomplete_event_on_disconnect(self._billing_source) + if self._billing_source is source_iterator: + return current + primary: Final = partial_responses_incomplete_event_on_disconnect(source_iterator) + if current is None or primary is None: + return current or primary + if primary.response.usage is not None: + _combine_responses_fallback_usage(current, primary.response.usage) + return current + + def __aiter__(self) -> "FallbackResponsesStreamWrapper": + return self + + @property + def yielded_first_chunk(self) -> bool: + return self._yielded_first_chunk + + async def __anext__(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # yields what the stream yields + try: + chunk: Final = await self._async_generator.__anext__() + except StopAsyncIteration: + if self.completed_response is None: + self.completed_response = getattr(source_iterator, "completed_response", None) + raise + if self.completed_response is None and getattr(chunk, "type", None) in terminal_event_types: + self.completed_response = chunk # pyright: ignore[reportAttributeAccessIssue] # the terminal event + self._yielded_first_chunk = True + sequence_number: Final = getattr(chunk, "sequence_number", None) + if not isinstance(sequence_number, int): + return chunk + if not self.fallback_headers_adopted: + self._last_sequence_number = max(self._last_sequence_number, sequence_number) + return chunk + self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number) + if self._last_sequence_number == sequence_number: + return chunk + return chunk.model_copy(update={"sequence_number": self._last_sequence_number}) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # a response event, the only item with a sequence_number + + async def aclose(self) -> None: + await self._async_generator.aclose() + + def background_stream_can_release(item: object) -> bool: + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + if ( + type(source_iterator) is not ResponsesAPIStreamingIterator + or source_iterator.custom_llm_provider != "openai" + ): + return False + if not _responses_event_announces_background_run(item): + return False + return not attempt.can_fall_back() + + async def stream_with_fallbacks() -> AsyncGenerator[object, None]: + held: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback + response_id_exposed = False # rebind-ok: set once, before an early-released event is yielded + try: + async for item in _areported( + attempt, + cast("AsyncIterator[object]", source_iterator), # cast-ok: an async stream + lambda last: _responses_total_tokens(last, source_iterator), + ): + if not response_id_exposed and background_stream_can_release(item): + response_id_exposed = True + if not response_id_exposed and _responses_stream_holds_event(item, len(held)): + held = (*held, item) + continue + for held_event in held: + yield held_event + held = () + yield item + for held_event in held: + yield held_event + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + source_iterator.model or "", + source_iterator.custom_llm_provider or "", + wrapper.yielded_first_chunk, + ) + if _is_context_window(stream_error) + else cast("MidStreamFallbackError", stream_error) # cast-ok: not a context window error + ) + if response_id_exposed: + _raise_original_mid_stream_error(e) + async with contextlib.aclosing( + _responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held) + ) as fallback_stream: + async for fallback_item in fallback_stream: + yield fallback_item + except Exception: + for held_event in held: + yield held_event + raise + finally: + with anyio.CancelScope(shield=True): + if hasattr(source_iterator, "aclose"): + try: + await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except Exception as exc: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing source: %s", exc) + + wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks()) + return wrapper + + +def _responses_continuation_request( + attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator" +) -> Mapping[str, object] | None: + """The request a fallback continues from once a Responses stream broke after text, or None before + output. Raises the primary's error when the output is constrained, a tool call began, a content + filter cut it, or the output ceiling is spent.""" + from litellm.litellm_core_utils.token_counter import ( + token_counter, # pyright: ignore[reportUnknownVariableType] # untyped tokenizer argument + ) + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + RESPONSES_CONTINUATION_KWARG, + ResponsesContinuation, + constrains_responses_continuation, + continuation_output_ceilings, + responses_continuation_input, + ) + + if _mid_stream_error_before_output(e): + return None + if source_iterator.started_tool_call or isinstance( + e.original_exception, litellm_exceptions.ContentPolicyViolationError + ): + _raise_original_mid_stream_error(e) + cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") + if constrains_responses_continuation(_params_sent_to_deployment(attempt, cut_deployment_id)): + _raise_original_mid_stream_error(e) + request: Final = attempt.request + earlier: Final = request.get(RESPONSES_CONTINUATION_KWARG) + continuation: Final = ( + earlier.extended(e.generated_content) + if isinstance(earlier, ResponsesContinuation) + else ResponsesContinuation(request.get("input"), e.generated_content) + ) + emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content) + reduced: Final = continuation_output_ceilings(request, emitted_tokens) + if reduced is None: + _raise_original_mid_stream_error(e) + return { + **request, + **reduced, + RESPONSES_CONTINUATION_KWARG: continuation, + "input": responses_continuation_input(continuation), + } + + +async def _responses_fallback_attempt( + attempt: StreamAttempt, + e: MidStreamFallbackError, + source_iterator: "BaseResponsesAPIStreamingIterator", + adopt_headers: Callable[[object], _Prepared], + held: tuple[object, ...], +) -> AsyncGenerator[object, None]: + """Re-enters the fallback walk for a mid-stream Responses error. The primary's held lifecycle events + reach the client only when no fallback lands.""" + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ResponsesContinuationSeam + + continued: Final = _responses_continuation_request(attempt, e, source_iterator) + seam: Final = ResponsesContinuationSeam(e.generated_content if continued is not None else "") + partial_usage: Final = _extract_partial_responses_usage(source_iterator) + trigger: Final = _fallback_trigger( + e, (litellm_exceptions.ContentPolicyViolationError, litellm_exceptions.ContextWindowExceededError) + ) + fallback_response: object = None # rebind-ok: set once a fallback answered, closed at the end + fallback_yielded = False # rebind-ok: flipped on the first fallback item + try: + fallback_response = await attempt.fallback(trigger, continued) # rebind-ok: rebound as the stream advances + prepared: Final = adopt_headers(fallback_response) + if hasattr(fallback_response, "__aiter__"): + spliced: Final = cast( # cast-ok: the fallback's events + "AsyncIterator[object]", + seam.splice(fallback_response), # pyright: ignore[reportArgumentType] # an async stream + ) + async for fallback_item in spliced: + _apply_fallback_hidden_params_to_item(fallback_item, prepared) + if partial_usage is not None: + _combine_responses_fallback_usage(fallback_item, partial_usage) + fallback_yielded = True + yield fallback_item + else: + fallback_yielded = True # rebind-ok: rebound as the stream advances + yield fallback_response + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Responses streaming fallback also failed: %s", fallback_error) + if not fallback_yielded: + for held_event in held: + yield held_event + _raise_fallback_failure(fallback_error) + finally: + if fallback_response is not None and hasattr(fallback_response, "aclose"): + with anyio.CancelScope(shield=True): + try: + await fallback_response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except Exception as exc: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing fallback: %s", exc) + + +# --- the attempt hook --------------------------------------------------------------------------- + + +async def open_attempt_stream(attempt: StreamAttempt, response: object) -> object: + """What an attempt returns once its stream is wrapped: a chat stream's first chunk is read here, so + a stream that fails before any token raises inside the attempt.""" + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + if attempt.op == "acompletion" and isinstance(response, CustomStreamWrapper): + if response.completion_stream is None and response.make_call is not None: # pyright: ignore[reportUnknownMemberType] # untyped attributes + await response.fetch_stream() + first_chunk: Final = await _aread_first_chunk(response) + held_slot: Final = attempt.slot.pop_all() if attempt.slot is not None else contextlib.ExitStack() + return wrap_chat_stream(attempt, response, first_chunk, held_slot) + if not attempt.kwargs.get("stream"): + return response + if attempt.op == "aresponses" and isinstance(response, BaseResponsesAPIStreamingIterator): + return wrap_responses_stream(attempt, response) + if attempt.op in ("aanthropic_messages", "anthropic_messages") and hasattr(response, "__aiter__"): + return wrap_anthropic_stream(attempt, cast("AsyncIterator[bytes]", response)) # cast-ok: a byte stream + return response + + +def open_attempt_stream_sync(attempt: StreamAttempt, response: object) -> object: + if attempt.op == "completion" and isinstance(response, CustomStreamWrapper): + if response.completion_stream is None and response.make_call is not None: # pyright: ignore[reportUnknownMemberType] # untyped attributes + response.fetch_sync_stream() + return wrap_chat_stream_sync(attempt, response, _read_first_chunk(response)) + return response + + +def shape_attempt_kwargs(kwargs: Mapping[str, object], deployment: Mapping[str, object]) -> Mapping[str, object]: + """A Responses continuation is shaped for the deployment the fallback landed on.""" + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + RESPONSES_CONTINUATION_KWARG, + responses_continuation_input_for, + ) + + if RESPONSES_CONTINUATION_KWARG not in kwargs: + return kwargs + shaped: Final = without(kwargs, RESPONSES_CONTINUATION_KWARG) + shaped["input"] = responses_continuation_input_for(kwargs, deployment) + return shaped + + +__all__ = [ + "MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS", + "MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS", + "FallbackAwareAnthropicMessagesStream", + "FallbackAwareStreamWrapper", + "StreamAttempt", + "open_attempt_stream", + "open_attempt_stream_sync", + "shape_attempt_kwargs", +] diff --git a/litellm/router_rust/invoke/walk.py b/litellm/router_rust/invoke/walk.py new file mode 100644 index 00000000000..4dab8edc07f --- /dev/null +++ b/litellm/router_rust/invoke/walk.py @@ -0,0 +1,398 @@ +""" +The request metadata the Python router stamps as it walks retries and fallbacks, kept for a +``RustCore`` call. router-core decides the walk and reports it (each attempt's ``scope``, +``max_retries`` and ``model_group_size``, and the ``fallback`` and ``retry_logged`` events); this +module only holds what each level's metadata dict holds in the Python router and lays it over the +attempt's kwargs. + +In the Python router one metadata dict serves a level's retries, and every fallback hop copies +its parent's (``run_async_fallback``'s ``_hop_metadata``), so a hop's breadcrumbs never reach +its parent. Each scope here is one such dict. The root level's dict is the caller's own (a proxy +reads a failed request's walk off it), so a fresh call's root writes reach that dict as well. +""" + +from collections.abc import Callable, Coroutine, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + from litellm.router_api import RoutingContext + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + +# `RETRY_BREADCRUMB_LIMIT`: how many failures `previous_models` keeps. +RETRY_BREADCRUMB_LIMIT: Final = 4 + +# The keys the walk stamps on each attempt's metadata. +_WALK_KEYS: Final = ( + "original_model_group", + "attempted_fallbacks", + "attempted_retries", + "max_retries", + "model_group_size", + "previous_models", + "request_retry_count", +) +# What a breadcrumb reads from the last attempt a scope made. +_SERVED_KEYS: Final = ("model_info", "deployment") +# The walk state `run_async_fallback` hands every hop as kwargs. +_HOP_KWARGS: Final = ("fallback_depth", "max_fallbacks", "attempted_targets", "include_fallback_errors") + + +class _Scope: + __slots__ = ("kwargs", "metadata") + + def __init__( + self, + metadata: dict[str, object], # mutable-ok: the level's metadata, stamped as the walk runs + kwargs: Mapping[str, object], + ) -> None: + self.metadata: Final = metadata + self.kwargs: Final = kwargs + + +def _breadcrumb( + metadata: Mapping[str, object], model: str, error: BaseException, params_base_model: Callable[[str], str | None] +) -> dict[str, object]: # mutable-ok: a fresh RetryAttemptRecord + """``_core_log_retry``'s record of the failure.""" + from litellm.litellm_core_utils.get_litellm_params import get_base_model_from_litellm_call_metadata + + model_info: Final = as_str_mapping(metadata.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + known_id: Final = deployment_id if isinstance(deployment_id, str) else None + base_model: Final = get_base_model_from_litellm_call_metadata(metadata=metadata) or ( + params_base_model(known_id) if known_id is not None else None + ) + deployment: Final = metadata.get("deployment") + attempted: Final = metadata.get("attempted_retries") + return { + "model_group": model, + "deployment_id": known_id, + "model": deployment if isinstance(deployment, str) else None, + "base_model": base_model, + "exception_type": type(error).__name__, + "exception_string": str(error), + "attempted_retries": attempted if type(attempted) is int else None, + } + + +def _kept_breadcrumbs(earlier: object) -> tuple[object, ...]: + """The breadcrumbs a new one joins: the latest ``RETRY_BREADCRUMB_LIMIT - 1``.""" + if not isinstance(earlier, list | tuple): + return () + stamped: Final[tuple[object, ...]] = tuple(earlier) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # the breadcrumbs as stamped + return stamped[-(RETRY_BREADCRUMB_LIMIT - 1) :] + + +def log_retry_into( + kwargs: dict[str, object], # mutable-ok: the request kwargs whose metadata the record joins + error: Exception, + params_base_model: Callable[[str], str | None], +) -> dict[str, object]: # mutable-ok: the same kwargs, as ``_core_log_retry`` returns them + """ + ``_core_log_retry`` (``RoutingContext.log_retry``): the failure joins the request metadata's + ``previous_models`` and counts toward ``request_retry_count``, with a ``litellm.routing.retry`` + phase event when a pre-routing strategy picked the group. + """ + from litellm.integrations.otel.runtime import phase_event + + field: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + metadata: Final = kwargs[field] + view: Final = as_str_mapping(metadata) + if view is None: + # Python reads it with ``.get``. + raise AttributeError(f"{type(metadata).__name__!r} object has no attribute 'get'") + model_group: Final = kwargs.get("model") + record: Final = _breadcrumb(view, "", error, params_base_model) + record["model_group"] = model_group if isinstance(model_group, str) else None + deployment_id: Final = record["deployment_id"] + breadcrumbs: Final = (*_kept_breadcrumbs(view.get("previous_models")), record) + earlier: Final = view.get("request_retry_count") + request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1 + decision: Final = view.get("routing_decision") + router_name: Final = (as_str_mapping(decision) or {}).get("router_model_name") + if isinstance(router_name, str): + phase_event( + "litellm.routing.retry", + { + "litellm.routing.router_model_name": router_name, + "litellm.retry.count": request_retry_count, + "error.type": type(error).__name__, + **({"litellm.deployment.model_group": model_group} if isinstance(model_group, str) else {}), + **({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}), + }, + ) + metadata["previous_models"] = breadcrumbs # pyright: ignore[reportIndexIssue] # the logging object already holds this dict + metadata["request_retry_count"] = request_retry_count # pyright: ignore[reportIndexIssue] # same dict, read by the cap + return kwargs + + +class WalkMetadata: + """The metadata each level of one call's walk holds, by scope.""" + + __slots__ = ("_attempted", "_field", "_live", "_params_base_model", "_request", "_scopes") + + def __init__( + self, + request: Mapping[str, object], + field: str, + params_base_model: Callable[[str], str | None], + *, + live: bool = False, + ) -> None: + model: Final = request.get("model") + root: Final[dict[str, object]] = {"attempted_fallbacks": 0} # mutable-ok: the root level's metadata + if model is not None: + root["original_model_group"] = model + self._request: Final = request + self._field: Final = field + self._params_base_model: Final = params_base_model + self._scopes: Final[dict[int, _Scope]] = {0: _Scope(root, {})} # mutable-ok: filled as the walk hops + self._attempted: AttemptedFallbackTargets | None = None + # A call whose retry and fallback wrappers stamp the request itself; a stream's recovery + # re-enters below them. + self._live: Final = live + if live: + self._stamp_fallback_entry() + + def _stamp_fallback_entry(self) -> None: + """The head of ``async_function_with_fallbacks``, in the request's own buckets. ``run_async_fallback``'s + hops carry their ``attempted_targets`` and never reach it.""" + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + + request: Final = self._request + if isinstance(request.get("attempted_targets"), AttemptedFallbackTargets): + return + sibling: Final = request.get("metadata" if self._field == "litellm_metadata" else "litellm_metadata") + if isinstance(sibling, dict): + sibling.pop("attempted_fallbacks", None) # pyright: ignore[reportUnknownMemberType] # the caller's metadata dict + sibling.pop("original_model_group", None) # pyright: ignore[reportUnknownMemberType] # see above + bucket: Final = request.get(self._field) + if not isinstance(bucket, dict): + return + bucket["attempted_fallbacks"] = 0 + model: Final = request.get("model") + if model is not None: + bucket["original_model_group"] = model + + def _retry_bucket(self, scope: _Scope) -> dict[str, object] | None: # mutable-ok: the request's own dict + """``_metadata`` of ``async_function_with_retries`` when it is a dict of the request's: the + root level's, or a hop's when the bucket is not the one ``run_async_fallback`` copies.""" + if not self._live: + return None + name: Final = "litellm_metadata" if "litellm_metadata" in self._request else "metadata" + if scope is not self._scopes[0] and name == self._field: + return None + bucket: Final = self._request.get(name) + # `... or {}`: an empty bucket leaves the stamps on a dict of their own. + return bucket if isinstance(bucket, dict) and bucket else None # pyright: ignore[reportUnknownVariableType] # the request's untyped metadata dict + + def _write_through(self, scope: _Scope, values: Mapping[str, object]) -> None: + bucket: Final = self._retry_bucket(scope) + if bucket is not None: + bucket.update(values) + + def retries_opened(self, event: Mapping[str, object]) -> None: + """The head of ``async_function_with_retries``: the group's size and the retry tracking, before any pick.""" + scope: Final = self._scope(event.get("scope")) + stamped: Final = {"attempted_retries": 0, "max_retries": event.get("max_retries")} + scope.metadata.update(stamped) + self._write_through(scope, stamped) + bucket: Final = self._retry_bucket(scope) + if bucket is not None and isinstance(bucket.get("model_group"), str): + bucket["model_group_size"] = event.get("model_group_size") + + def retry_budget(self, event: Mapping[str, object]) -> None: + """``_metadata["max_retries"] = num_retries``, once a level's first failure settled its overrides.""" + scope: Final = self._scope(event.get("scope")) + stamped: Final = {"max_retries": event.get("max_retries")} + scope.metadata.update(stamped) + self._write_through(scope, stamped) + + def _scope(self, value: object) -> _Scope: + found: Final = self._scopes.get(value) if isinstance(value, int) else None + return found if found is not None else self._scopes[0] + + def targets(self) -> "AttemptedFallbackTargets": + """The walk's ``attempted_targets``: the targets tried and the ones skipped, and why.""" + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + + if self._attempted is None: + self._attempted = AttemptedFallbackTargets() + return self._attempted + + def hop(self, event: Mapping[str, object]) -> None: + """A ``fallback`` event: the new level copies its parent's metadata and kwargs.""" + parent: Final = self._scope(event.get("parent")) + attempted: Final = self.targets() + for key in (event.get("from"), event.get("to")): + if isinstance(key, str): + attempted.record(key) + metadata: Final = dict(parent.metadata) + original: Final = metadata.pop("original_model_group", self._request.get("model")) + metadata.pop("model_group", None) + metadata.pop("attempted_fallbacks", None) + metadata["original_model_group"] = original + # A dict target's non-string `model` is what `kwargs["model"]` holds on the hop. + metadata["model_group"] = event["unroutable"] if "unroutable" in event else event.get("to") + metadata["attempted_fallbacks"] = event.get("depth") + kwargs: Final = { + **parent.kwargs, + "fallback_depth": event.get("depth"), + "max_fallbacks": event.get("max_fallbacks"), + "attempted_targets": attempted, + } + if self._request.get("include_fallback_errors") is True: + kwargs["include_fallback_errors"] = True + scope: Final = event.get("scope") + if isinstance(scope, int): + self._scopes[scope] = _Scope(metadata, kwargs) + + def hop_kwargs(self, event: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a fresh dict for the loggers + """The kwargs the level held after a hop (``kwargs["model"] = target``), as the + fallback loggers see them.""" + scope: Final = self._scope(event.get("scope")) + model: Final = scope.metadata.get("model_group", event.get("to")) + return {**self._request, **scope.kwargs, "model": model, self._field: dict(scope.metadata)} + + def log_retry(self, event: Mapping[str, object], error: BaseException | None) -> None: + """A ``retry_logged`` event: the failure joins the scope's ``previous_models``.""" + scope: Final = self._scope(event.get("scope")) + metadata: Final = scope.metadata + attempted: Final = event.get("attempted_retries") + if isinstance(attempted, int): + metadata["attempted_retries"] = attempted + self._write_through(scope, {"attempted_retries": attempted}) + if error is None: + return + record: Final = _breadcrumb(metadata, str(event.get("model")), error, self._params_base_model) + earlier: Final = metadata.get("previous_models") + kept: Final = _kept_breadcrumbs(earlier) + count: Final = metadata.get("request_retry_count") + metadata["previous_models"] = (*kept, record) + metadata["request_retry_count"] = (count if type(count) is int and count >= 0 else 0) + 1 + self._write_through( + scope, + {"previous_models": metadata["previous_models"], "request_retry_count": metadata["request_retry_count"]}, + ) + + def deployment_that_just_failed(self, step: Mapping[str, object]) -> str | None: + """``_deployment_that_just_failed``: the deployment the metadata an attempt inherits names, + before the attempt stamps its own; ``None`` when the walk's level has run none yet.""" + model_info: Final = as_str_mapping(self._scope(step.get("scope")).metadata.get("model_info")) + model_id: Final = model_info.get("id") if model_info is not None else None + return model_id if isinstance(model_id, str) else None + + def request_retry_count(self, step: Mapping[str, object]) -> object: + """The level's ``request_retry_count``, as the retry loop has counted it so far.""" + return self._scope(step.get("scope")).metadata.get("request_retry_count") + + def stamp(self, step: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The attempt's kwargs with its level's walk metadata and hop kwargs laid over them.""" + scope: Final = self._scope(step.get("scope")) + metadata: Final = scope.metadata + metadata["attempted_retries"] = step.get("retry") + metadata["max_retries"] = step.get("max_retries") + metadata["model_group_size"] = step.get("model_group_size") + self._write_through( + scope, {"attempted_retries": metadata["attempted_retries"], "max_retries": metadata["max_retries"]} + ) + served: Final = dict(as_str_mapping(kwargs.get(self._field)) or {}) + served.update({key: metadata[key] for key in _WALK_KEYS if key in metadata}) + metadata.update({key: served.get(key) for key in _SERVED_KEYS}) + return {**kwargs, **{key: scope.kwargs[key] for key in _HOP_KWARGS if key in scope.kwargs}, self._field: served} + + +async def _log_failed_fallbacks(ctx: "RoutingContext", detail: str, parent_otel_span: "Span | None") -> None: + from litellm._logging import verbose_router_logger + from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments_with_debug_info + + cooldown_info: Final = await async_get_cooldown_deployments_with_debug_info( + ctx=ctx, parent_otel_span=parent_otel_span + ) + verbose_router_logger.error( + "litellm.router.py::async_function_with_fallbacks() - " + "Error occurred while trying to do fallbacks - %s\n" + "Debug Information:\nCooldown Deployments=%s", + detail, + cooldown_info, + ) + + +class FallbackOutcomes: + """What ``run_async_fallback`` does when a hop returns: it logs the hop to the process's + custom loggers and, under ``include_fallback_errors``, lists the level's errors (its own + first, then each hop that failed) in the response's ``x-litellm-fallback-errors``.""" + + __slots__ = ("_failed", "_header_errors", "_include", "_pending") + + def __init__(self, request: Mapping[str, object]) -> None: + self._include: Final = request.get("include_fallback_errors", False) is True + self._failed: Final[dict[object, list[object]]] = {} # mutable-ok: each level's failed hops, in order + self._header_errors: Final[list[list[object]]] = [] # mutable-ok: each successful hop's errors, innermost first + self._pending: Final[list[Coroutine[object, object, None]]] = [] # mutable-ok: logger calls not yet awaited + + def done( + self, + event: Mapping[str, object], + level_error: BaseException | None, + hop_error: BaseException | None, + kwargs: Mapping[str, object], + ) -> None: + from litellm.litellm_core_utils.add_retry_fallback_headers import get_fallback_error_info + from litellm.router_utils import fallback_event_handlers as handlers + + if level_error is None or not isinstance(level_error, Exception): + return + parent: Final = event.get("parent") + failed: Final = self._failed.setdefault(parent, []) + original: Final = str(event.get("from")) + if event.get("succeeded") is True: + if self._include: + self._header_errors.append([get_fallback_error_info(level_error), *failed]) + self._pending.append( + handlers.log_success_fallback_event( # pyright: ignore[reportUnknownMemberType] # takes the kwargs as an untyped dict + original_model_group=original, kwargs=dict(kwargs), original_exception=level_error + ) + ) + return + if isinstance(hop_error, Exception): + failed.append(get_fallback_error_info(hop_error)) + self._pending.append( + handlers.log_failure_fallback_event( # pyright: ignore[reportUnknownMemberType] # see above + original_model_group=original, kwargs=dict(kwargs), original_exception=level_error + ) + ) + + def failed_fallbacks(self, ctx: "RoutingContext", detail: str, parent_otel_span: "Span | None") -> None: + """The ``except`` around the regular fallbacks of ``async_function_with_fallbacks_common_utils``: + ``detail`` is the failure's text as it read then, the cooldowns are read when the pending calls run.""" + self._pending.append(_log_failed_fallbacks(ctx, detail, parent_otel_span)) + + def take_pending(self) -> tuple[Coroutine[object, object, None], ...]: + pending: Final = tuple(self._pending) + self._pending.clear() + return pending + + def add_headers(self, response: object, attempted_fallbacks: int) -> object: + """``add_fallback_headers_to_response`` for each successful hop, innermost first.""" + from litellm.litellm_core_utils.add_retry_fallback_headers import add_fallback_headers_to_response + + headed: object = response # rebind-ok: gains each level's errors + for errors in self._header_errors: + headed = add_fallback_headers_to_response( + response=headed, + attempted_fallbacks=attempted_fallbacks, + fallback_errors=errors, # pyright: ignore[reportArgumentType] # FallbackErrorInfo dicts + ) + return headed + + +async def await_all(pending: Sequence[Coroutine[object, object, None]]) -> None: + for call in pending: + await call + + +__all__ = ["RETRY_BREADCRUMB_LIMIT", "FallbackOutcomes", "WalkMetadata", "await_all"] diff --git a/litellm/router_rust/invoke/websocket.py b/litellm/router_rust/invoke/websocket.py new file mode 100644 index 00000000000..e7baeb92345 --- /dev/null +++ b/litellm/router_rust/invoke/websocket.py @@ -0,0 +1,127 @@ +""" +Responses websocket turns on the Rust router core (``async_admit_deployment_turn``): the socket is +opened on one deployment, and the core hands the handler a ``turn_admission`` callable that re-runs +that deployment's admission before every later turn: the io-token turn budget, the deployment +budget limiter, the routing strategy (on that deployment alone) and the pre-call checks. + +Which strategy and selector apply is the Rust core's routing context (``routing_context``); the +selectors, limiters and checks are the live Python objects the host keeps. +""" + +from collections.abc import Callable, Mapping +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.core_helpers import without +from litellm.router_rust.invoke.shaping import as_dict + +if TYPE_CHECKING: + from litellm.router_rust.invoke.endpoints import EndpointCore + + +def turn_admission( + core: "EndpointCore", model_group: str, deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> Callable[[], object]: + """The ``turn_admission`` the handler calls before each later turn.""" + return partial(admit_deployment_turn, core, model_group, deployment, kwargs) + + +def _deployment_as_configured_now(core: "EndpointCore", deployment: Mapping[str, object]) -> Mapping[str, object]: + model_info: Final = deployment.get("model_info") + model_id: Final = as_dict(model_info).get("id") if isinstance(model_info, dict) else None # pyright: ignore[reportUnknownArgumentType] # the configured model_info dict + current: Final = core.configured_deployment(model_id) if isinstance(model_id, str) else None + return deployment if current is None else current.model_dump(exclude_none=True) + + +def _budget_limiter(core: "EndpointCore") -> object: + """The router's deployment-budget callback; the Rust core declines every config that has one.""" + from litellm.router_rust.core import RustCoreUnsupported + + try: + return core.registry().plugins("budget_limiter") + except RustCoreUnsupported: + return None + + +async def admit_deployment_turn( + core: "EndpointCore", model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] +) -> Mapping[str, object]: + """`async_admit_deployment_turn`.""" + from litellm import exceptions as litellm_exceptions + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + async_admit_io_token_turn, + without_io_token_limits, + ) + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + from litellm.utils import get_utc_datetime + + deployment: Final = _deployment_as_configured_now(core, opened_on) + rate_limit_check: Final = next( + (callback for callback in core.routing_callbacks() if isinstance(callback, ModelRateLimitingCheck)), + None, + ) + io_token_admission: Final[Mapping[str, object]] = ( + MappingProxyType({}) + if rate_limit_check is None + else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime()) # pyright: ignore[reportArgumentType] # the configured deployment dict + ) + budget_limiter: Final[object] = _budget_limiter(core) + if budget_limiter is not None: + try: + await budget_limiter.async_filter_deployments( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # RouterBudgetLimiting + model=model_group, + healthy_deployments=[dict(deployment)], + messages=None, + request_kwargs=dict(request_kwargs), + ) + except ValueError as e: + raise litellm_exceptions.RateLimitError( + message=str(e), + llm_provider="", + model=model_group, + category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + ) from e + await _admit_turn_through_routing_strategy(core, model_group, deployment, request_kwargs) + await core.router.async_routing_strategy_pre_call_checks( + deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None + ) + return io_token_admission + + +async def _admit_turn_through_routing_strategy( + core: "EndpointCore", model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object] +) -> None: + """`_admit_turn_through_routing_strategy`: the group's strategy picks among this deployment alone.""" + from litellm import exceptions as litellm_exceptions + from litellm.router_api import RoutingContext + from litellm.router_rust.steps.selection import pick_async + from litellm.router_utils.handle_error import async_raise_no_deployment_exception + + routing_kwargs: Final = without(request_kwargs, "litellm_logging_obj") + override: Final = routing_kwargs.get("routing_strategy") + strategy, scope, group = core.handle.routing_context(model_group, override if isinstance(override, str) else None) + selectors: Final = core.selectors + selector: Final = selectors.selector_for(strategy, scope, group, routing_kwargs) + if strategy == "simple-shuffle" or selector is None: + return + selected: Final = await pick_async( + strategy, selector, model_group, [dict(deployment)], None, None, dict(routing_kwargs) + ) + if selected is None: + no_deployment: Final = await async_raise_no_deployment_exception( + ctx=RoutingContext(core), # pyright: ignore[reportArgumentType] # the error reads registry() and health() + model=model_group, + parent_otel_span=None, + ) + raise litellm_exceptions.RateLimitError( + message=str(no_deployment), + llm_provider="", + model=model_group, + category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + ) from no_deployment + if selectors.is_override(strategy, selector): + await selector.async_pre_call_check(dict(deployment), None) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # every selector is a CustomLogger + + +__all__ = ["admit_deployment_turn", "turn_admission"] diff --git a/litellm/router_rust/steps/README.md b/litellm/router_rust/steps/README.md new file mode 100644 index 00000000000..9b5c68bdba9 --- /dev/null +++ b/litellm/router_rust/steps/README.md @@ -0,0 +1,12 @@ +# steps + +Answers to router-core's host steps that need live Python objects. router-core makes the routing decision; these modules supply the Python-only facts or run the Python-only objects it asks about. + +- `selection`: stateful strategies (latency, tpm/rpm, cost, usage) on `select` steps +- `prerouting`: auto, complexity, adaptive and quality routers, and session binding, on `pre_route` steps +- `precall`: token counts, context windows and param support for pre-call checks +- `plugins`: `Router(plugins=[...])` objects +- `callbacks`: the optional pre-call checks (budgets, affinity, rate limits) as `CustomLogger`s +- `access`: the caller facts read from auth metadata + +A Rust host answers these steps natively, or declines the config at build. diff --git a/litellm/router_rust/steps/__init__.py b/litellm/router_rust/steps/__init__.py new file mode 100644 index 00000000000..c47505e7468 --- /dev/null +++ b/litellm/router_rust/steps/__init__.py @@ -0,0 +1 @@ +"""Answers to router-core host steps that need live Python objects.""" diff --git a/litellm/router_rust/steps/access.py b/litellm/router_rust/steps/access.py new file mode 100644 index 00000000000..087a87e2c6b --- /dev/null +++ b/litellm/router_rust/steps/access.py @@ -0,0 +1,85 @@ +"""The caller facts the Rust core routes on, read from the request's auth metadata: whether the +caller is a proxy admin, the models its key and team allow (which narrow an access-group-only +caller to its groups' deployments) and, for a proxy admin, the model the request names.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request, request_user_api_key_auth + +_NAMES: Final = TypeAdapter(tuple[object, ...]) + + +def _names(value: object) -> tuple[str, ...]: + names: Final = _NAMES.validate_python(value) if isinstance(value, list | tuple | set | frozenset) else () + return tuple(str(name) for name in names) + + +def _admin_model(request: Mapping[str, object]) -> object: + """The model ``filter_team_based_models`` compares team deployments against.""" + metadata: Final = as_str_mapping(request.get("metadata")) or {} + litellm_metadata: Final = as_str_mapping(request.get("litellm_metadata")) or {} + return request.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group") + + +def caller_fields(request: Mapping[str, object]) -> Mapping[str, object]: + """``proxy_admin``, ``allowed_models`` and ``admin_model`` for the Rust request document.""" + auth: Final[object] = request_user_api_key_auth(request) + if auth is None: + return {} + allowed: Final = sorted({*_names(getattr(auth, "models", None)), *_names(getattr(auth, "team_models", None))}) + admin: Final = is_proxy_admin_request(request) + model: Final = _admin_model(request) if admin else None + return { + **({"allowed_models": allowed} if allowed else {}), + **({"proxy_admin": True} if admin else {}), + **({"admin_model": model} if isinstance(model, str) else {}), + } + + +def _router_weights_error(raw: object) -> BaseException: + """The error ``validate_router_weights`` raises for weights the Rust core refused.""" + from pydantic import ValidationError + + from litellm.types.router_weights import validate_router_weights + + try: + _ = validate_router_weights(raw) + except ValidationError as error: + return error + return RuntimeError(f"router-core refused router weights the validator accepts: {raw!r}") + + +def selection_refusal(step: Mapping[str, object], model: str, request: Mapping[str, object]) -> BaseException | None: + """The error for a pick the Rust core refused over who the caller is or what it asked for: + a public name several teams share, deployments reserved for other teams, invalid request + router weights or a non-numeric deployment weight. None for any other failure.""" + import litellm.exceptions as litellm_exceptions + + kind: Final = step.get("error_kind") + named: Final = str(step.get("error_model") or model) + if kind == "multiple_teams": + return litellm_exceptions.BadRequestError( + message=( + f"Model name '{named}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=named, + llm_provider="", + ) + if kind == "reserved": + return litellm_exceptions.BadRequestError( + message=f"Deployment {named} is reserved for another team until {step.get('error_detail')}", + model=named, + llm_provider="", + ) + if kind == "invalid_router_weights": + return _router_weights_error(request.get("_router_weights")) + if kind == "non_numeric_weight": + return TypeError(f"Deployment {step.get('error_detail')} must be numeric") + return None diff --git a/litellm/router_rust/steps/callbacks.py b/litellm/router_rust/steps/callbacks.py new file mode 100644 index 00000000000..b906a496550 --- /dev/null +++ b/litellm/router_rust/steps/callbacks.py @@ -0,0 +1,650 @@ +""" +Host-side optional pre-call checks for ``RustCore``: the ``CustomLogger`` checks a router builds +from ``optional_pre_call_checks``, ``provider_budget_config``, per-deployment budgets and +``model_group_affinity_config``, registered in the host's ``callbacks`` list exactly as the Python +router's ``add_optional_pre_call_checks`` registers them, since they are features users configure. + +The Rust core asks the host to run every routing callback's ``async_filter_deployments`` through a +``callback_filters`` select stage (``async_callback_filter_deployments``) and decides itself what +the surviving deployments or the raised error mean: a pick with nothing left, a retry, a fallback. + +This module also keeps the process-wide holder rules (``litellm/router_utils/global_callback_holders``): +a retiring router hands the checks it vacates to a survivor, a runtime removal keeps a check +another live router still has, and ``observe(request_scoped=True)`` takes every routing callback +off the host lists while the router keeps running them (``_routing_callbacks``). It also answers +which outcome callbacks the Python router would still have running for a request +(``OptionalChecks.outcome_callbacks``), which router-core records by. It never reaches a Python +router core. +""" + +from collections.abc import Iterable, Mapping, Sequence +from itertools import chain +from typing import TYPE_CHECKING, Final +from weakref import WeakSet + +from litellm._logging import verbose_router_logger +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import PROMPT_CACHE_PIN_CACHE_MAX_SIZE +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.routing_common_utils import get_request_team_id +from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_strategy.least_busy import LeastBusyLoggingHandler +from litellm.router_utils.global_callback_holders import ( + live_check_owners, + routers_holding_global_callbacks, + routers_holding_global_callbacks_lock, +) +from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( + DeploymentAffinityCheck, + completed_affinity_response, + warn_on_unknown_model_group_affinity_flags, +) +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.router_python_host import PythonHost + from litellm.router_rust.steps.selection import StrategySelectors + from litellm.router_rust.surface.registry import HostRegistry + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.router_utils.global_callback_holders import GlobalCallbackHolder + from litellm.types.router import Deployment + +_AFFINITY_CHECKS: Final = ( + "deployment_affinity", + "responses_api_deployment_check", + "session_affinity", + "encrypted_content_affinity", +) +# `_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS`: the checks a runtime list drops when it leaves them out. +_RUNTIME_TOGGLEABLE: Final[Mapping[str, type[CustomLogger]]] = { + "prompt_caching": PromptCachingDeploymentCheck, + "enforce_model_rate_limits": ModelRateLimitingCheck, + "encrypted_content_affinity": EncryptedContentAffinityCheck, +} + + +def _callbacks(host: "PythonHost") -> list[object]: # mutable-ok: the process list itself + return host.callback_list("callbacks") # pyright: ignore[reportArgumentType] # not in the CallbackList literal + + +def register_router_selector(host: "PythonHost", selector: CustomLogger) -> None: + """``_register_router_selector``: least-busy also counts calls as they start.""" + if isinstance(selector, LeastBusyLoggingHandler): + if isinstance(host.callback_list("input_callback"), list): # pyright: ignore[reportUnnecessaryIsInstance] # a process list a user may have reassigned + host.callbacks.add_litellm_input_callback(selector) + else: + host.set_callback_list("input_callback", [selector]) + if isinstance(_callbacks(host), list): # pyright: ignore[reportUnnecessaryIsInstance] # see above + host.callbacks.add_litellm_callback(selector) + + +def _move_before_deployment_affinity( + callback_list: list[object], # mutable-ok: reordered in place + callback: object, +) -> None: + if callback not in callback_list: + return + callback_list.remove(callback) + index: Final = next( + (i for i, entry in enumerate(callback_list) if isinstance(entry, DeploymentAffinityCheck)), + len(callback_list), + ) + callback_list.insert(index, callback) + + +class OptionalChecks: + """One router's optional pre-call checks (``optional_callbacks``) and the stores they read.""" + + def __init__( + self, + owner: "GlobalCallbackHolder", + host: "PythonHost", + cache: DualCache, + hosted: "HostRegistry", + ctx: object, + arguments: Mapping[str, object], + ) -> None: + self._owner: Final = owner + self._host: Final = host + self._cache: Final = cache + self._hosted: Final = hosted + self._ctx: Final = ctx + self.optional_callbacks: list[object] | None = None # mutable-ok: the router's live list, as Python keeps it + self.router_budget_logger: RouterBudgetLimiting | None = None + self.provider_budget_config: object = arguments.get("provider_budget_config") + ttl: Final = arguments.get("deployment_affinity_ttl_seconds") + self.deployment_affinity_ttl_seconds: object = 3600 if ttl is None else ttl + self.session_affinity_repin_on_failover: object = bool(arguments.get("session_affinity_repin_on_failover")) + affinity_config: Final = arguments.get("model_group_affinity_config") + self.model_group_affinity_config: dict[str, list[str]] | None = ( # mutable-ok: the argument as passed + affinity_config if isinstance(affinity_config, dict) else None # pyright: ignore[reportAttributeAccessIssue] # the argument as passed + ) + warn_on_unknown_model_group_affinity_flags(self.model_group_affinity_config) + self.prompt_caching_pin_cache: DualCache = DualCache( + redis_cache=cache.redis_cache, + in_memory_cache=InMemoryCache(max_size_in_memory=PROMPT_CACHE_PIN_CACHE_MAX_SIZE), + ) + self._request_scoped_selectors: tuple[CustomLogger, ...] = () + self._request_scoped_policies: tuple[CustomLogger, ...] = () + # ``_slack_alerting_loggers``: the Slack alerting logger ``alerting_config`` builds, if any. + self.slack_alerting_loggers: tuple[CustomLogger, ...] = () + # Whether the outcome callbacks would still be on the host lists (``_release_global_callbacks``). + self._outcome_callbacks_listed: bool = True + # The logging objects ``_core_bind_to_request_logging`` would have bound them to. + self._bound_requests: Final[WeakSet[Logging]] = WeakSet() + + # -- construction --------------------------------------------------------------------------- + + def build(self, checks: object, model_list: object) -> None: + """The constructor's checks: a budget appends ``router_budget_limiting`` to the list given (in + place, as Python does), the list is added, a per-group affinity config adds the affinity + check, and ``apply_default_settings`` adds the empty default list.""" + listed: list[object] | None = checks if isinstance(checks, list) else None # pyright: ignore[reportUnknownVariableType] # mutable-ok: appended in place, as Python does # rebind-ok: the argument as passed + if RouterBudgetLimiting.should_init_router_budget_limiter( # pyright: ignore[reportUnknownMemberType] # untyped model_list + model_list=model_list if isinstance(model_list, list) else None, # pyright: ignore[reportUnknownArgumentType] # the argument as passed + provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed + ): + if listed is not None: + listed.append("router_budget_limiting") + else: + listed = ["router_budget_limiting"] # rebind-ok: the list Python builds + if listed is not None: + self.add(listed) # pyright: ignore[reportUnknownArgumentType] # names as given + if self.model_group_affinity_config: + self.ensure_deployment_affinity_callback() + self.add([]) + + def _is_priority_group(self, model: str) -> bool: + """``_is_priority_routing_group``.""" + hosted: Final = self._hosted + resolved: Final = hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # the host registry's alias lookup + group: Final = hosted.get_routing_group(resolved) + return group is not None and group.routing_strategy == "priority" + + async def repin_session_after_success( + self, litellm_params: Mapping[str, object], response: object, cooldowns: "CooldownCache" + ) -> None: + """``_repin_session_after_success``: with ``session_affinity_repin_on_failover`` on, a completed + success moves the session's pin to the deployment that served it, among the group's + deployments not cooling down.""" + if not self.session_affinity_repin_on_failover: + return + try: + if not completed_affinity_response(response): + return + callback: Final = next( + (logger for logger in self.optional_callbacks or () if isinstance(logger, DeploymentAffinityCheck)), + None, + ) + if callback is None: + return + model_group: Final = callback.get_repin_model_group(litellm_params) + if model_group is None or self._is_priority_group(model_group): + return + registered_ids: Final = self._hosted._core_get_candidate_model_ids_for_route( # pyright: ignore[reportPrivateUsage] # the host registry's candidate lookup + model=model_group, team_id=get_request_team_id(litellm_params) + ) + cooldown_ids: Final = frozenset( + model_id + for model_id, _ in await cooldowns.async_get_active_cooldowns( + model_ids=list(registered_ids), parent_otel_span=None + ) + ) + await callback.async_repin_session_after_success(litellm_params, model_group, registered_ids - cooldown_ids) + except Exception as error: # noqa: BLE001 # optional pin updates must not interrupt success accounting + verbose_router_logger.debug("Session re-pinning failed: %s", error) + + def _affinity_check(self, user_key: bool, responses_api: bool, session_id: bool) -> DeploymentAffinityCheck: + return DeploymentAffinityCheck( + cache=self._cache, + ttl_seconds=self.deployment_affinity_ttl_seconds, # pyright: ignore[reportArgumentType] # the argument as passed + enable_user_key_affinity=user_key, + enable_responses_api_affinity=responses_api, + enable_session_id_affinity=session_id, + model_group_affinity_config=self.model_group_affinity_config, + is_priority_group=self._is_priority_group, + session_affinity_repin_on_failover=self.session_affinity_repin_on_failover, # pyright: ignore[reportArgumentType] # same + ) + + def _own(self, callback: CustomLogger) -> None: + if self.optional_callbacks is None: + self.optional_callbacks = [] + self.optional_callbacks.append(callback) + self._host.callbacks.add_litellm_callback(callback) + + def ensure_deployment_affinity_callback(self) -> None: + """``_ensure_deployment_affinity_callback``: the affinity check with every global flag off.""" + if any(isinstance(cb, DeploymentAffinityCheck) for cb in self.optional_callbacks or ()): + return + self._own(self._affinity_check(False, False, False)) + + def _add_affinity(self, checks: Sequence[object]) -> None: + user_key: Final = "deployment_affinity" in checks + responses_api: Final = "responses_api_deployment_check" in checks + session_id: Final = "session_affinity" in checks + if not (user_key or responses_api or session_id): + return + if self.optional_callbacks is None: + self.optional_callbacks = [] + existing: Final = next( + (cb for cb in self.optional_callbacks if isinstance(cb, DeploymentAffinityCheck)), + None, + ) + if existing is None: + self._own(self._affinity_check(user_key, responses_api, session_id)) + return + existing.enable_user_key_affinity = existing.enable_user_key_affinity or user_key + existing.enable_responses_api_affinity = existing.enable_responses_api_affinity or responses_api + existing.enable_session_id_affinity = existing.enable_session_id_affinity or session_id + existing.ttl_seconds = self.deployment_affinity_ttl_seconds # pyright: ignore[reportAttributeAccessIssue] # the argument as passed + if self.model_group_affinity_config: + existing.model_group_affinity_config = self.model_group_affinity_config + + def _add_encrypted_content_affinity(self, enable_global_affinity: bool) -> None: + """``_add_encrypted_content_affinity_check``.""" + if not ( + enable_global_affinity + or EncryptedContentAffinityCheck.has_model_group_affinity_enabled(self.model_group_affinity_config) + ): + return + if self.optional_callbacks is None: + self.optional_callbacks = [] + existing: Final = next( + (cb for cb in self.optional_callbacks if isinstance(cb, EncryptedContentAffinityCheck)), None + ) + if existing is not None: + existing.ctx = self._ctx # pyright: ignore[reportAttributeAccessIssue] # the core's RoutingContext serves registry() + existing.enable_global_affinity = existing.enable_global_affinity or enable_global_affinity + existing.model_group_affinity_config = self.model_group_affinity_config or {} + check: EncryptedContentAffinityCheck = existing # rebind-ok: the check to move + else: + check = EncryptedContentAffinityCheck( # rebind-ok: the check to move + ctx=self._ctx, # pyright: ignore[reportArgumentType] # see above + enable_global_affinity=enable_global_affinity, + model_group_affinity_config=self.model_group_affinity_config, + ) + self._own(check) + _move_before_deployment_affinity(self.optional_callbacks, check) + _move_before_deployment_affinity(_callbacks(self._host), check) + + def add(self, checks: Sequence[object] | None) -> None: + """``add_optional_pre_call_checks``.""" + if checks is None: + return + self._add_affinity(checks) + self._add_encrypted_content_affinity("encrypted_content_affinity" in checks) + for name in checks: + if name in _AFFINITY_CHECKS: + continue + callback: CustomLogger | None = None + if name == "prompt_caching": + callback = PromptCachingDeploymentCheck( + cache=self.prompt_caching_pin_cache, is_priority_group=self._is_priority_group + ) + elif name == "router_budget_limiting": + if self.budget_limiter() is not None: + continue + callback = RouterBudgetLimiting( + dual_cache=self._cache, + provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed + model_list=self._hosted._core_model_list, # pyright: ignore[reportPrivateUsage, reportArgumentType] # the live model_list + ) + self.router_budget_logger = callback + elif name == "enforce_model_rate_limits": + callback = ModelRateLimitingCheck(dual_cache=self._cache) + if callback is None: + continue + if self.optional_callbacks is not None and any( + isinstance(existing, type(callback)) for existing in self.optional_callbacks + ): + continue + self._own(callback) + + def set(self, checks: object) -> None: + """``set_optional_pre_call_checks``: drop the toggleable checks the list leaves out, then add it.""" + if checks is None: + return + listed: Final[tuple[object, ...]] = tuple(checks) if isinstance(checks, list | tuple) else () # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # names as given + requested: Final = frozenset(listed) + for name, callback_cls in _RUNTIME_TOGGLEABLE.items(): + if name not in requested: + self._remove_of_type(callback_cls) + self.add(listed) + + def _remove_of_type(self, callback_cls: type[CustomLogger]) -> None: + """``_remove_optional_callbacks_of_type``: kept on the host while any other live router of + the process has a check of that type.""" + if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks): + return + self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] + others: Final = tuple(owner for owner in live_check_owners if id(owner) != id(self._owner)) + if any(any(type(cb) is callback_cls for cb in (owner.optional_callbacks or ())) for owner in others): + return + callbacks: Final = _callbacks(self._host) + for cb in tuple(callbacks): + if type(cb) is callback_cls: + self._host.callbacks.remove_callback_from_list_by_object(callbacks, cb, require_self=False) # pyright: ignore[reportUnknownMemberType] # the manager's untyped helper + + # -- budgets -------------------------------------------------------------------------------- + + def budget_limiter(self) -> RouterBudgetLimiting | None: + """``_core_get_router_deployment_budget_limiter``: exact type, so a proxy subclass is not taken.""" + if self.router_budget_logger is not None: + return self.router_budget_logger + for cb in self.optional_callbacks or (): + if isinstance(cb, RouterBudgetLimiting) and type(cb) is RouterBudgetLimiting: + self.router_budget_logger = cb + return cb + return None + + def sync_deployment_budget(self, deployment: "Deployment") -> None: + """``_sync_deployment_budget_config``, after a deployment is added.""" + model_id: Final = deployment.model_info.id + if model_id is None: + return + limiter: RouterBudgetLimiting | None = self.budget_limiter() # rebind-ok: built when first needed + params: Final = deployment.litellm_params + if params.max_budget is None or params.budget_duration is None: + if limiter is not None: + limiter.unregister_deployment_budget(model_id=model_id) + return + if limiter is None: + self.add(["router_budget_limiting"]) + limiter = self.budget_limiter() # rebind-ok: the one just built + if limiter is not None: + limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) # pyright: ignore[reportUnknownMemberType] # pydantic's untyped dump + + def unregister_deployment_budget(self, model_id: str) -> None: + limiter: Final = self.budget_limiter() + if limiter is not None: + limiter.unregister_deployment_budget(model_id=model_id) + + # -- the routing callbacks ------------------------------------------------------------------ + + def loggers(self) -> tuple[CustomLogger, ...]: + """``_optional_loggers``.""" + return tuple(cb for cb in self.optional_callbacks or () if isinstance(cb, CustomLogger)) + + def _policies_not_set_globally(self) -> tuple[CustomLogger, ...]: + keys: Final = self._host.callbacks.get_custom_logger_key + global_keys: Final = frozenset(keys(cb) for cb in _callbacks(self._host) if isinstance(cb, CustomLogger)) + return tuple(policy for policy in self._request_scoped_policies if keys(policy) not in global_keys) + + def bind_to_request_logging(self, logging_obj: "Logging") -> None: + """``_core_bind_to_request_logging``'s routing half: the request-scoped selectors replace the + host's same-keyed callbacks for the request, and the policies the host does not run are + added to it. The outcome callbacks are not bound: the Rust core records every attempt's + outcome itself, so the request is noted for ``outcome_callbacks`` instead.""" + self._bound_requests.add(logging_obj) + for selector in self._request_scoped_selectors: + logging_obj.replace_global_callback(selector) + for policy in self._policies_not_set_globally(): + logging_obj.add_dynamic_callback(policy) + + def outcome_callbacks( + self, request: Mapping[str, object], *, sync: bool, make_call: bool = True + ) -> Mapping[str, bool]: + """Router-core's ``OutcomeCallbacks`` for ``request``, empty (its default) while the outcome + callbacks are listed. Past that only the request's bound logging object runs them, and + ``_core_bind_to_request_logging`` binds neither ``sync_deployment_callback_on_success`` nor any + success callback a sync call runs. ``make_call``: the attempt runs through ``make_call``.""" + if self._outcome_callbacks_listed: + return {} + bound: Final = request.get("litellm_logging_obj") in self._bound_requests + return {"success": bound and not sync, "failure": bound, "make_call": make_call} + + def routing_callbacks(self) -> Iterable[object]: + """``_routing_callbacks``: the host's callbacks, or after ``observe(request_scoped=True)`` the + host's callbacks with the router's own selectors and policies in place of theirs.""" + if not (self._request_scoped_selectors or self._request_scoped_policies): + return tuple(_callbacks(self._host)) + keys: Final = self._host.callbacks.get_custom_logger_key + replaced: Final = frozenset(map(keys, self._request_scoped_selectors)) + return ( + *self._host.callbacks.without_custom_logger_keys(_callbacks(self._host), replaced), # pyright: ignore[reportArgumentType] # the process list holds any callback + *self._request_scoped_selectors, + *self._policies_not_set_globally(), + ) + + async def filter_deployments( + self, + model: str, + deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: object, + request: Mapping[str, object], + ) -> list[object]: # mutable-ok: as the filters return it + """``async_callback_filter_deployments``: each routing callback narrows the deployments in + turn; a filter's error is raised for the Rust core to decide on.""" + remaining: list[object] = list(deployments) # mutable-ok: filter contract # rebind-ok: narrowed + base: Final[object] = CustomLogger.async_filter_deployments # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts + request_kwargs: Final = dict(request) + for callback in tuple(self.routing_callbacks()): + # The base filter returns its input unchanged: only overriding filters are run. + if not isinstance(callback, CustomLogger) or type(callback).async_filter_deployments is base: # pyright: ignore[reportUnknownMemberType] # see above + continue + remaining = await callback.async_filter_deployments( # pyright: ignore[reportUnknownMemberType, reportAssignmentType, reportUnknownVariableType] # see above + model=model, + healthy_deployments=remaining, + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + request_kwargs=request_kwargs, + parent_otel_span=None, + ) + return remaining + + # -- release -------------------------------------------------------------------------------- + + def serve_through_request_logging(self, selectors: Sequence[CustomLogger], dropped: Sequence[object]) -> None: + """``_core_serve_through_request_logging``: keep running ``selectors`` (and request-scoped + adaptive hooks) and the optional checks, but take everything off the host lists.""" + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + + adaptive: Final = tuple( + AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) # pyright: ignore[reportArgumentType] # an adaptive router + for tagged in chain.from_iterable(self._hosted._core_adaptive_routers.values()) # pyright: ignore[reportPrivateUsage] # the host registry's live registry + ) + self._request_scoped_selectors = (*selectors, *adaptive) + self._request_scoped_policies = (*self.loggers(), *self.slack_alerting_loggers) + self.release(dropped) + + def own_adaptive_hooks(self) -> tuple[CustomLogger, ...]: + """``_adaptive_router_hooks``: the host's adaptive post-call hooks of this router's routers.""" + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + + own: Final = frozenset( + id(tagged.strategy) + for tagged in chain.from_iterable(self._hosted._core_adaptive_routers.values()) # pyright: ignore[reportPrivateUsage] # see above + ) + return tuple( + hook + for hook in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) + if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own + ) + + def retire_state(self, selectors: Sequence[object]) -> None: + """``retire_routing_state``: stop the background work of the selectors and the budget limiter.""" + for strategy in (*selectors, *self.loggers()): + if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting): + strategy.retire() + + def release(self, dropped: Sequence[object]) -> None: + """``_release_global_callbacks``: drop ``dropped``, the optional checks and the adaptive hooks + from the host lists, then hand each vacated check to a surviving router's same-keyed one.""" + self._outcome_callbacks_listed = False + with routers_holding_global_callbacks_lock: + routers_holding_global_callbacks.discard(self._owner) + adaptive: Final = self.own_adaptive_hooks() + slack: Final = self.slack_alerting_loggers + vacated: Final = self._host.callbacks.drop_callbacks( + (*dropped, *(self.optional_callbacks or ()), *slack, *adaptive), method_owners=(self._owner, *slack) + ) + with routers_holding_global_callbacks_lock: + survivors: Final = tuple(routers_holding_global_callbacks) + candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) # pyright: ignore[reportPrivateUsage] # the holder protocol + for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated): + register_router_selector(self._host, candidate) + adaptive_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, adaptive)) + survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None) # pyright: ignore[reportPrivateUsage] # see above + if survivor is not None and vacated & adaptive_keys: + survivor._sync_adaptive_router_hooks() # pyright: ignore[reportPrivateUsage] # see above + + +_ENCRYPTED_PIN: Final = "_encrypted_content_affinity_pinned" + + +def _request_with_failed_ids(request: Mapping[str, object], step: Mapping[str, object]) -> Mapping[str, object]: + """The request as the filters read it in the Python router: the Rust walk holds what the Python + one keeps in the attempt's kwargs, ``_excluded_deployment_ids`` from a weighted failover hop, + ``_retry_skipped_deployment_ids`` from the retry loop (``deployment_ids_failed_in_request``) and + ``_target_order`` from an order fallback.""" + excluded: Final = tuple(str(id) for id in _ids_of(step.get("excluded_ids"))) + skipped: Final = tuple(str(id) for id in _ids_of(step.get("skipped_ids"))) + target_order: Final = step.get("target_order") + if not excluded and not skipped and target_order is None: + return request + return { + **request, + **({"_excluded_deployment_ids": list(excluded)} if excluded else {}), + **({"_retry_skipped_deployment_ids": skipped} if skipped else {}), + **({"_target_order": target_order} if target_order is not None else {}), + } + + +def _ids_of(value: object) -> Sequence[object]: + return value if isinstance(value, list | tuple) else () # pyright: ignore[reportUnknownVariableType] # the step's id list + + +def filtered_ids(remaining: Iterable[object], candidates: Sequence[str]) -> list[str]: # mutable-ok: a JSON list + """The ids of the deployments the filters left, in their order, among the candidates.""" + ids: Final[list[str]] = [] # mutable-ok: built here + for deployment in remaining: + found = (as_str_mapping((as_str_mapping(deployment) or {}).get("model_info")) or {}).get("id") + if isinstance(found, str) and found in candidates and found not in ids: + ids.append(found) + return ids + + +class RustOptionalChecks: + """``RustCore`` members for its optional pre-call checks and routing callbacks.""" + + # RustCore sets these as Final attributes in its constructor. + @property + def _opt(self) -> OptionalChecks: + checks: Final[OptionalChecks] = self.__dict__["_checks"] # pyright: ignore[reportAny] # RustCore's own attribute + return checks + + @property + def _sel(self) -> "StrategySelectors": + selectors: Final[StrategySelectors] = self.__dict__["_selectors"] # pyright: ignore[reportAny] # same + return selectors + + @property + def _reg(self) -> "HostRegistry": + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # same + return hosted + + @property + def optional_callbacks(self) -> "list[object] | None": # mutable-ok: the router's live list + return self._opt.optional_callbacks + + @property + def router_budget_logger(self) -> RouterBudgetLimiting | None: + return self._opt.router_budget_logger + + @property + def deployment_affinity_ttl_seconds(self) -> object: + return self._opt.deployment_affinity_ttl_seconds + + @property + def session_affinity_repin_on_failover(self) -> object: + return self._opt.session_affinity_repin_on_failover + + @property + def model_group_affinity_config(self) -> "dict[str, list[str]] | None": # mutable-ok: the argument as passed + return self._opt.model_group_affinity_config + + def add_optional_pre_call_checks(self, optional_pre_call_checks: "Sequence[object] | None") -> None: + self._opt.add(optional_pre_call_checks) + + def apply_default_settings(self) -> None: + """Add the (empty) default checks; an enabled continuation registers the prefill check.""" + from litellm.router_rust.invoke.streaming import register_continuation_prefill_check + + self._opt.add([]) + if getattr(self, "enable_mid_stream_fallback_continuation", False): + register_continuation_prefill_check(self._opt._host) # pyright: ignore[reportPrivateUsage] # the checks' host + + # The members another router's retire reads (`global_callback_holders`). + + def _global_callback_candidates(self) -> tuple[CustomLogger, ...]: + return (*self._sel.callback_selectors(), *self._opt.loggers(), *self._opt.slack_alerting_loggers) + + @property + def _core_adaptive_routers(self) -> Mapping[str, object]: + return self._reg._core_adaptive_routers # pyright: ignore[reportPrivateUsage] # the host registry's live registry + + def _sync_adaptive_router_hooks(self) -> None: + strategies: Final = self._reg.strategies + if strategies is not None: + strategies._sync_adaptive_router_hooks() # pyright: ignore[reportPrivateUsage] # the strategies' own hook sync + + def _join_callback_holders(self) -> None: + with routers_holding_global_callbacks_lock: + routers_holding_global_callbacks.add(self) + live_check_owners.add(self) + + def outcome_callbacks( + self, request: Mapping[str, object], *, sync: bool, make_call: bool = True + ) -> Mapping[str, bool]: + return self._opt.outcome_callbacks(request, sync=sync, make_call=make_call) + + def _serve_through_request_logging(self) -> None: + selectors: Final = self._sel + self._opt.serve_through_request_logging( + selectors.callback_selectors(), (*selectors.callback_selectors(), *selectors.override_selectors()) + ) + + def _retire_callbacks(self) -> None: + """``_core_discard``'s callback half: retire the routing state, release the callbacks. The + selectors stay: a retired router still picks with them.""" + live_check_owners.discard(self) + selectors: Final = (*self._sel.callback_selectors(), *self._sel.override_selectors()) + self._opt.retire_state(selectors) + self._opt.release(selectors) + + async def _answer_callback_filters( + self, step: Mapping[str, object], request: Mapping[str, object], messages: object + ) -> tuple[object, BaseException | None]: + """A ``callback_filters`` step: the remaining ids, or a filter's error as an attempt + failure with the error itself, which the caller keeps to raise.""" + from litellm.router_rust.core import attempt_failure + + candidates: Final = [str(id) for id in step.get("candidates") or ()] # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType, reportUnknownArgumentType] # the step's id list + records: Final = self._records_of(candidates) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # RustCore's deployment dicts + narrowed: Final = _request_with_failed_ids(request, step) + try: + remaining: Final = await self._opt.filter_deployments( + str(step.get("model_group")), + records, # pyright: ignore[reportUnknownArgumentType] # see above + messages, + narrowed, + ) + except Exception as error: # noqa: BLE001 # handed to the Rust core as the try's failure + return attempt_failure(error), error + # A filter's pin lands on the request kwargs the pick goes on to read. + if narrowed is not request and narrowed.get(_ENCRYPTED_PIN) and isinstance(request, dict): + request[_ENCRYPTED_PIN] = True # rebind-ok: the request dict the pick goes on to read + return filtered_ids(remaining, candidates), None # pyright: ignore[reportUnknownArgumentType] # see above + + +__all__ = [ + "OptionalChecks", + "RustOptionalChecks", + "filtered_ids", + "register_router_selector", +] diff --git a/litellm/router_rust/steps/plugins.py b/litellm/router_rust/steps/plugins.py new file mode 100644 index 00000000000..4af73313838 --- /dev/null +++ b/litellm/router_rust/steps/plugins.py @@ -0,0 +1,87 @@ +""" +Host side of the Rust core's routing-plugin pipeline (``Router(plugins=[...])``). + +The Rust core decides where the pipeline applies (only the async pick runs it; the sync pick +refuses a router with plugins) and which deployments the plugins' candidate models keep +(``router-core/src/routing_plugins.rs``). The plugins themselves are live Python objects, so this +module runs them on a ``routing_plugins`` select step, the way the Python router core's +``_run_routing_plugins`` does, and reports the candidate models left on the request. +""" + +from collections.abc import Mapping, Sequence +from typing import Final, Protocol + +from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias +from litellm.types.router import RoutingContext, RoutingPlugin + +SYNC_PATH_MESSAGE: Final = ( + "Router(plugins=[...]) is configured, but this call resolved to the synchronous " + "deployment-selection path, which never runs the routing-plugin pipeline. This " + "happens for sync Router methods (e.g. Router.completion()) and for async calls " + "with a routing_strategy that has no async-native selector (e.g. legacy " + "'usage-based-routing', v1). Silently skipping " + "configured plugins would let a policy plugin (e.g. a deny-all rule) be bypassed. " + "Use an async Router method with a supported routing_strategy (simple-shuffle, " + "usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), " + "or remove `plugins` from the Router config." +) +_CANDIDATES_KEY: Final = "_routing_plugin_candidate_models" + + +class PluginRegistry(Protocol): + def _core_get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> Sequence[Mapping[str, object]] | None: ... + + +def _resolved_litellm_models( + registry: PluginRegistry, model: str +) -> list[str]: # mutable-ok: the plugins' candidate list + """``_core_resolved_litellm_models``: the provider models a call to ``model`` is served by.""" + return [ + litellm_model + for deployment in registry._core_get_model_list(model_name=model) or () # pyright: ignore[reportPrivateUsage] # the host registry's listing + if isinstance(litellm_model := (as_str_mapping(deployment.get("litellm_params")) or {}).get("model"), str) + and litellm_model + ] + + +def candidate_models(request: Mapping[str, object]) -> Mapping[str, object]: + """The answer to a ``routing_plugins`` step: the candidate models the plugins left on the + request, ``None`` when none narrowed it.""" + metadata: Final = as_str_mapping(request.get(get_metadata_variable_name_from_kwargs(dict(request)))) + return {"models": metadata.get(_CANDIDATES_KEY) if metadata is not None else None} + + +async def run_plugins( + plugins: Sequence[RoutingPlugin], + registry: PluginRegistry, + model_group_alias: Mapping[str, object], + model: str, + request: dict[str, object], # mutable-ok: the plugins stamp their outcome on the caller's request + messages: object, +) -> None: + """``_run_routing_plugins``: run ``plugins`` in order over ``model``'s candidate models, then + stamp their signals (and a narrowed candidate list) on the request's metadata.""" + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + registered: Final = resolve_model_group_alias(model_group_alias, model) or model + models: Final = _resolved_litellm_models(registry, registered) + metadata_key: Final = get_metadata_variable_name_from_kwargs(request) + metadata: Final = request.setdefault(metadata_key, {}) + raw_messages: Final[object] = messages or list[object]() + context = RoutingContext( # rebind-ok: each plugin returns the context the next one reads + raw_messages=raw_messages, # pyright: ignore[reportArgumentType] # the request's messages + structured_messages=resolve_structured_messages(messages=messages, request_kwargs=request) or [], # pyright: ignore[reportArgumentType] # the request's messages + candidate_models=list(models), + metadata=metadata, # pyright: ignore[reportArgumentType] # the request's metadata dict + ) + for plugin in plugins: + context = await plugin.run(context) + metadata["routing_plugin_signals"] = context.signals # pyright: ignore[reportIndexIssue] # the request's metadata dict + if len(context.candidate_models) < len(models): + metadata[_CANDIDATES_KEY] = context.candidate_models # pyright: ignore[reportIndexIssue] # see above + + +__all__ = ["SYNC_PATH_MESSAGE", "PluginRegistry", "candidate_models", "run_plugins"] diff --git a/litellm/router_rust/steps/precall.py b/litellm/router_rust/steps/precall.py new file mode 100644 index 00000000000..156c7b5e20f --- /dev/null +++ b/litellm/router_rust/steps/precall.py @@ -0,0 +1,494 @@ +""" +Host-side facts for the Rust core's ``enable_pre_call_checks`` filter. + +The Rust core decides which deployments the pre-call checks drop and which error a group with +nothing left raises (``router-core/src/precall.rs``). What needs live Python objects is answered +here, on a ``pre_call`` select step: the prompt's token count (messages, Responses input and +instructions, tools and an Anthropic ``system`` block), each deployment's context window from +its router model info, and which requested params each deployment takes. Ported from the Python +router core's ``_pre_call_checks`` helpers; it never reaches a Python router core. + +``pre_call_checks`` is the one decision made here: ``RoutingContext.pre_call_checks`` hands over +deployment records rather than a step, and those need not be in the Rust registry. +""" + +from collections.abc import Callable, Mapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol + +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider +from litellm.litellm_core_utils.token_counter import ( + offload_token_count, + served_openai_models, + single_encoding_openai_model, + split_embedding_input_tokens, +) +from litellm.router_python_host import python_host +from litellm.router_strategy.complexity_router.context_compaction import compaction_pending +from litellm.types.router import LiteLLM_Params + +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + from litellm.types.utils import LlmProviders + +# The chat params a deployment must take when the request sends them (`_PRE_CALL_CHECK_FILTERED_PARAMS`). +_FILTERED_PARAMS: Final = frozenset({"response_format"}) + + +class PreCallRegistry(Protocol): + """The host registry reads the facts need.""" + + def _core_get_model_info(self, id: str) -> Mapping[str, object] | None: ... # the registry's name + + def _core_deployments_named(self, model_name: str) -> Sequence[Mapping[str, object]] | None: ... + + def _core_get_router_model_info( + self, deployment: Mapping[str, object], received_model_name: str + ) -> Mapping[str, object]: ... + + +def count_tokens( + host: "PythonHost", + messages: object, + input: object, # the request field's name + request_kwargs: Mapping[str, object] | None, + tokenizer_model: str, + deployments: Sequence[Mapping[str, object]] = (), +) -> int: + """``_count_pre_call_check_tokens``: messages, or Responses input normalized to messages, plus + the tools and Anthropic ``system`` block the message list never carries. Embedding media an + item of ``deployments`` reads as media is left out of the input: it never counts as text.""" + from litellm.llms.anthropic.messages.native.utils import anthropic_system_to_openai_message + + extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({}) + raw_instructions: Final = extras.get("instructions") + instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None + raw_tools: Final = extras.get("tools") + tools: Final = raw_tools if _is_list(raw_tools) and raw_tools else None + system_message: Final = anthropic_system_to_openai_message(extras.get("system")) + counter: Final = host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # the host's overloads name litellm's untyped returns + if messages is not None: + counted_messages: Final = (system_message, *_items(messages)) if system_message is not None else messages + return counter(model=tokenizer_model, messages=counted_messages, tools=tools) + if input is not None: + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments)) + input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped + input=text_input, # pyright: ignore[reportArgumentType] # str | list, as the request carries it + responses_api_request={"instructions": instructions} if instructions is not None else {}, + only_since_compaction=True, + ) + return counter(model=tokenizer_model, messages=input_messages, tools=tools) + raise ValueError("Either messages or input must be provided to count tokens") + + +def _max_input_tokens(registry: PreCallRegistry, model: str, deployment: Mapping[str, object]) -> int | None: + """``_deployment_max_input_tokens``: the declared context window, or None when there is none or + the deployment cannot be resolved.""" + try: + model_info: Final = registry._core_get_router_model_info( # pyright: ignore[reportPrivateUsage] # the host registry's lookup + deployment=deployment, received_model_name=model + ) + except Exception as e: # noqa: BLE001 # best-effort: an unmappable deployment must not hide the others + verbose_router_logger.debug("litellm.router.py::_deployment_max_input_tokens: skipping deployment. Got - %s", e) + return None + max_input_tokens: Final = model_info.get("max_input_tokens") + return max_input_tokens if isinstance(max_input_tokens, int) else None + + +def _items(value: object) -> tuple[object, ...]: + """The items of a list, tuple or set; nothing for any other value.""" + return tuple(value) if isinstance(value, list | tuple | set | frozenset) else () # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # the items are read as objects + + +def _is_list(value: object) -> bool: + return isinstance(value, list) + + +def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: + declared: Final = params.get("allowed_openai_params") + return frozenset(entry for entry in _items(declared) if isinstance(entry, str)) + + +def _capability_model_and_provider(deployment_params: Mapping[str, object], group: str) -> tuple[str, str]: + model_label: Final = str(deployment_params.get("model") or group) + custom_llm_provider: Final = deployment_params.get("custom_llm_provider") + authenticating_provider: Final = declared_authenticating_provider( + model_label, + custom_llm_provider, # pyright: ignore[reportArgumentType] # passed as configured, as the Python core does + ) + if authenticating_provider is not None: + return model_label.removeprefix(f"{authenticating_provider}/"), authenticating_provider + model, provider, _, _ = python_host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # see count_tokens + model=model_label, + custom_llm_provider=custom_llm_provider, + litellm_params=LiteLLM_Params.model_validate(dict(deployment_params)), + ) + return model, provider + + +def _supports_param(deployment: Mapping[str, object], group: str, param: str) -> bool: + """``_deployment_supports_param``.""" + deployment_params: Final = as_str_mapping(deployment.get("litellm_params")) + if not deployment_params: + return True + deployment_model_info: Final = as_str_mapping(deployment.get("model_info")) + base_model: Final = ( + deployment_model_info.get("base_model") if deployment_model_info else None + ) or deployment_params.get("base_model") + try: + model, custom_llm_provider = _capability_model_and_provider(deployment_params, group) + supported: Final = python_host.invoke("get_supported_openai_params")( # pyright: ignore[reportUnknownMemberType] # see count_tokens + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_supports_param: keeping %s for model=%s. Got - %s", param, group, e + ) + return True + return supported is None or param in supported + + +def _lacks_params(deployment: Mapping[str, object], group: str, params: frozenset[str]) -> bool: + return any(not _supports_param(deployment, group, param) for param in params) + + +def _rejects_params( + deployment: Mapping[str, object], group: str, params: frozenset[str], request_kwargs: Mapping[str, object] +) -> bool: + """``_deployment_rejects_params``: lacks a param neither the deployment nor the request drops or allows.""" + call_settings: Final[Mapping[str, object]] = { + **(as_str_mapping(deployment.get("litellm_params")) or {}), + **request_kwargs, + } + if call_settings.get("drop_params") is True: + return False + dropped: Final = call_settings.get("additional_drop_params") + dropped_names: Final = frozenset( + entry for entry in (_items(dropped) if _is_list(dropped) else ()) if isinstance(entry, str) + ) + return _lacks_params(deployment, group, params - _declared_param_allowlist(call_settings) - dropped_names) + + +def _requested_params(request_kwargs: Mapping[str, object]) -> frozenset[str]: + """``_pre_call_check_requested_chat_params``.""" + from litellm import utils as litellm_utils + + return _filtered_params( + litellm_utils.get_non_default_params(passed_params=dict(request_kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an untyped helper + ) + + +def _filtered_params(requested: Mapping[object, object]) -> frozenset[str]: + return frozenset( + param + for param, value in requested.items() + if isinstance(param, str) and param in _FILTERED_PARAMS and not isinstance(value, str) + ) + + +class PreCallFacts: + """The facts of one ``pre_call`` step.""" + + def __init__( + self, + host: "PythonHost", + registry: PreCallRegistry, + step: Mapping[str, object], + records: Callable[[str], Mapping[str, object]], + request: Mapping[str, object] | None, + messages: object, + input: object, # the request field's name + ) -> None: + self._host: Final = host + self._registry: Final = registry + self._model: Final = str(step.get("model_group")) + self._name: Final = str(step.get("model")) + self._candidates: Final = [str(id) for id in _items(step.get("candidates"))] + self._pool: Final = [str(id) for id in _items(step.get("pool"))] + self._records: Final = records + self._request: Final = request + self._messages: Final = messages + self._input: Final = input + self._has_input: Final = messages is not None or input is not None + self._windows: Final = {id: _max_input_tokens(registry, self._model, records(id)) for id in self._candidates} + + def _record(self, id: str) -> Mapping[str, object]: # a deployment id + return self._records(id) + + def _needs_count(self) -> bool: + return self._has_input and any(limit is not None for limit in self._windows.values()) + + def _tokenizer_model(self) -> str: + named: Final = self._registry._core_deployments_named(self._model) or () # pyright: ignore[reportPrivateUsage] # the host registry's index + return ( + single_encoding_openai_model( + served_openai_models(named) or served_openai_models(self._record(id) for id in self._candidates) + ) + or "" + ) + + def _candidate_records(self) -> Sequence[Mapping[str, object]]: + return tuple(self._record(id) for id in self._candidates) + + def _count(self) -> int | None: + try: + return count_tokens( + self._host, + self._messages, + self._input, + self._request, + self._tokenizer_model(), + self._candidate_records(), + ) + except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request + verbose_router_logger.error( + "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e + ) + return None + + async def _count_async(self) -> int | None: + try: + return await offload_token_count(count_tokens)( + self._host, + self._messages, + self._input, + self._request, + self._tokenizer_model(), + self._candidate_records(), + ) + except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request + verbose_router_logger.error( + "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e + ) + return None + + def _answer(self, input_tokens: int | None) -> Mapping[str, object]: + if not self._has_input: + return {"has_input": False} + request: Final[Mapping[str, object]] = self._request if self._request is not None else {} + params: Final = ( + _requested_params(request) + if self._request is not None and self._host.dynamic().drop_params is False + else frozenset[str]() + ) + deployments: Final[dict[str, dict[str, object]]] = {} # mutable-ok: built here, then returned + for id in dict.fromkeys([*self._candidates, *self._pool]): + record = self._record(id) + deployments[id] = { + "max_input_tokens": self._windows.get(id), + "lacks_params": bool(params) and id in self._candidates and _lacks_params(record, self._model, params), + "rejects_params": bool(params) and _rejects_params(record, self._model, params, request), + } + allowed_region: Final = request.get("allowed_model_region") + return { + "has_input": True, + "countable": not compaction_pending(self._request), + "input_tokens": input_tokens, + "allowed_model_region": allowed_region, + "requested_params": sorted(params), + "deployments": deployments, + } + + def answer(self) -> Mapping[str, object]: + """The facts on the sync path: the prompt is counted inline.""" + return self._answer(self._count() if self._needs_count() else None) + + async def answer_async(self) -> Mapping[str, object]: + """The facts on the async path: the prompt is counted off the event loop.""" + return self._answer(await self._count_async() if self._needs_count() else None) + + +def pre_call_checks( + host: "PythonHost", + registry: PreCallRegistry, + model: str, + deployments: Sequence[Mapping[str, object]], + messages: object, + input: object, # the request field's name + request_kwargs: Mapping[str, object] | None, + input_token_count: int | None = None, + skip_inline_token_count: bool = False, + request_deployments: Sequence[Mapping[str, object]] | None = None, +) -> Sequence[Mapping[str, object]]: + """``_pre_call_checks`` over the given deployments, as ``router-core/src/precall.rs`` decides it.""" + import litellm.exceptions as litellm_exceptions + + pool: Final = ( + tuple(request_deployments) + if request_deployments is not None + else (tuple(registry._core_deployments_named(model) or ()) or tuple(deployments)) # pyright: ignore[reportPrivateUsage] # the host registry's index + ) + candidates: Final = {f"candidate-{index}": record for index, record in enumerate(deployments)} + group: Final = {f"pool-{index}": record for index, record in enumerate(pool)} + records: Final = {**candidates, **group} + step: Final = {"model_group": model, "model": model, "candidates": list(candidates), "pool": list(group)} + facts: Final = PreCallFacts(host, registry, step, records.__getitem__, request_kwargs, messages, input) + counted: Final = ( + input_token_count + if input_token_count is not None or skip_inline_token_count or not facts._needs_count() # pyright: ignore[reportPrivateUsage] # this module's own class + else facts._count() # pyright: ignore[reportPrivateUsage] # see above + ) + answer: Final = facts._answer(counted) # pyright: ignore[reportPrivateUsage] # see above + rows: Final = as_str_mapping(answer.get("deployments")) or {} + allowed_region: Final = answer.get("allowed_model_region") + + def row(id: str) -> Mapping[str, object]: # a candidate or pool key + return as_str_mapping(rows.get(id)) or {} + + def over_window(id: str) -> bool: + limit: Final = row(id).get("max_input_tokens") + return isinstance(limit, int) and answer.get("countable") is True and counted is not None and counted > limit + + def out_of_region(id: str) -> bool: + params: Final = as_str_mapping(records[id].get("litellm_params")) or {} + return allowed_region is not None and params.get("region_name") != allowed_region + + too_small: Final = tuple(id for id in candidates if over_window(id)) + invalid: Final = frozenset(too_small) | frozenset(id for id in candidates if out_of_region(id)) + param_candidates: Final = tuple(id for id in candidates if id not in invalid) + lacking: Final = tuple(row(id).get("lacks_params") is True for id in param_candidates) + rejecting: Final = tuple(row(id).get("rejects_params") is True for id in param_candidates) + failing: Final = rejecting if lacking and all(lacking) else lacking + excluded: Final = invalid | frozenset(id for id, fails in zip(param_candidates, failing, strict=True) if fails) + if len(excluded) == len(candidates): + requested: Final = [str(param) for param in _items(answer.get("requested_params"))] + if not too_small and group and all(row(id).get("rejects_params") is True for id in group): + raise litellm_exceptions.UnsupportedParamsError( + message=( + f"no deployment of model={model} supports {requested}. " + "To drop these, set `litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, " + f"or send allowed_openai_params={requested} to pass them through" + ), + model=model, + ) + if too_small: + detail: Final = "".join( + f"Model={_error_model(records[id])}, Max Input Tokens={row(id).get('max_input_tokens')}, Got={counted}" + for id in too_small + ) + raise litellm_exceptions.ContextWindowExceededError( + message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context " + f"window large enough for this call.\n{detail}", + model=model, + llm_provider="", + ) + return tuple(record for id, record in candidates.items() if id not in excluded) + + +def _error_model(deployment: Mapping[str, object]) -> object: + """The name a context-window error gives a deployment: its base model, else its model.""" + params: Final = as_str_mapping(deployment.get("litellm_params")) or {} + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + return model_info.get("base_model") or params.get("base_model") or params.get("model") + + +def request_carries_image(messages: object, request: Mapping[str, object]) -> bool: + """``_request_carries_image``: the messages (or the request's own messages or input) hold an image.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + return request_contains_image_content(messages or request.get("messages") or request.get("input")) + + +def declares_no_vision(deployment: Mapping[str, object], group: str) -> bool: + """``_deployment_declares_no_vision``: ``model_info.supports_vision`` is ``False``, or, when it + is unset, the cost map says the model has vision explicitly disabled.""" + from litellm import utils as litellm_utils + + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + declared: Final = model_info.get("supports_vision") + if declared is not None: + return declared is False + params: Final = as_str_mapping(deployment.get("litellm_params")) or {} + if not params: + return False + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model, custom_llm_provider = _capability_model_and_provider(params, group) + return litellm_utils.is_vision_explicitly_disabled( + base_model if isinstance(base_model, str) else model, custom_llm_provider + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + +__all__ = [ + "PreCallFacts", + "PreCallRegistry", + "count_tokens", + "declares_no_vision", + "pre_call_checks", + "request_carries_image", +] + + +def _route_capability_resolvers() -> Mapping[str, Callable[[str, "LlmProviders"], object | None]]: + """``_ROUTE_CAPABILITY_RESOLVERS``: the calls only some providers serve, by handler name.""" + from litellm.utils import ProviderConfigManager + + return { + "adecisions": ProviderConfigManager.get_provider_decisions_config, # pyright: ignore[reportUnknownMemberType] # litellm's config lookup + "decisions": ProviderConfigManager.get_provider_decisions_config, # pyright: ignore[reportUnknownMemberType] # see above + } + + +# ``_DECISIONS_MEDIA_CALL_TYPES``: the calls whose audio or videos some deployments refuse. +DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"}) + + +def serves_route_capability(handler: str | None) -> bool: + """Whether only some providers serve the call ``handler`` names.""" + return handler is not None and handler in _route_capability_resolvers() + + +def cannot_serve_route(deployment: Mapping[str, object], group: str, handler: str) -> bool: + """Not ``_deployment_serves_route``: the deployment's provider has no config for the call.""" + from litellm.types.utils import LlmProviders + + resolver: Final = _route_capability_resolvers().get(handler) + params: Final = as_str_mapping(deployment.get("litellm_params")) + if resolver is None or params is None: + return False + try: + model, provider_name = _capability_model_and_provider(params, group) + provider: Final = next((member for member in LlmProviders if member.value == provider_name), None) + return provider is None or resolver(model, provider) is None + except Exception as e: # noqa: BLE001 # best-effort capability filter, as the Python core keeps it + verbose_router_logger.debug( + "litellm.router.py::_deployment_serves_route: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + +def refuses_decisions_media( + deployment: Mapping[str, object], group: str, sent_kwargs: Mapping[str, object], *, audio: bool, videos: bool +) -> bool: + """``_decisions_deployment_refuses_media``.""" + from litellm.decisions.call import deployment_refuses_media + + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + params: Final = as_str_mapping(deployment.get("litellm_params")) or {} + try: + model, custom_llm_provider = _capability_model_and_provider(params, group) + return deployment_refuses_media( + model, custom_llm_provider, model_info, {**params, **sent_kwargs}, audio=audio, videos=videos + ) + except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself + verbose_router_logger.debug( + "litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s", + group, + e, + ) + return False diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py new file mode 100644 index 00000000000..a83fbe29996 --- /dev/null +++ b/litellm/router_rust/steps/prerouting.py @@ -0,0 +1,1107 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportAny=false, reportExplicitAny=false +# (the ported core code reads the untyped strategy routers, LiteLLM_Params dict fields and request dicts) +""" +Host-side pre-routing for ``RustCore``. + +Before an async pick resolves its name, the Rust core asks the host to pre-route it (a +``pre_route`` step, ``router-core/src/prerouting.rs``): the complexity, auto, adaptive and quality +routers registered from ``auto_router/...`` marker deployments, and the Claude Code session +binding, may send the request to another model group. They are live Python objects, so they stay +host-side for now (bridged, to port): this module registers them from the host registry's +deployments the way the Python router's ``init_*_router_deployment``, +``_finalize_adaptive_router_if_configured`` and ``_unregister_pre_routing_strategy_for_deployment`` +do, and answers the step the way ``_core_async_pre_routing_hook`` and ``_pin_tier_params_onto_request`` +do. It never reaches a Python router core. +""" + +import json +import logging +import re +from collections.abc import Awaitable, Callable, Iterable, Mapping, MutableMapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast # noqa: TID251 # the strategy routers are untyped + +from litellm._internal_context import with_service_target +from litellm._logging import verbose_router_logger +from litellm.caching.caching import DualCache, InMemoryCache +from litellm.caching.redis_cache import log_redis_failure +from litellm.constants import ( + CLIENT_OUTPUT_CEILING_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, + OUTPUT_TOKEN_CEILING_PARAMS, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, +) +from litellm.integrations.otel.routing import routing_decision_attributes +from litellm.integrations.otel.runtime import phase_attributes +from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, without +from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference +from litellm.router_api import deployment_id +from litellm.router_host import router_host +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + AUTO_ROUTER_MODEL_PREFIX, + STRATEGY_ROUTER_PARAM_FIELDS, + GatedAutoRouterCapability, + capability_limit_violation, + claimed_capabilities, + classify_strategy_router_model, + count_capability_routers, +) +from litellm.router_strategy.complexity_router.context_compaction import ( + arm_compaction, + is_native_compaction_call, + reject_recursive_compactor, +) +from litellm.router_strategy.tag_based_routing import get_routing_tags_from_request_kwargs, is_valid_deployment_tag +from litellm.router_utils.fallback_event_handlers import record_pre_routing_selection +from litellm.types.router import ConsumedRequestTagsStamp, Deployment, LiteLLM_Params, TaggedPreRoutingStrategy +from litellm.types.utils import ( + PROMPT_QUOTING_ROUTING_DECISION_FIELDS, + CustomPricingLiteLLMParams, + StandardLoggingRoutingDecision, + all_litellm_params, +) +from litellm.utils import provider_rejectable_params + +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + from litellm.router_rust.surface.registry import HostRegistry + from litellm.router_strategy.auto_router.auto_router import PreRoutingHookResponse + +# The live registry dicts: model_name -> the strategies registered under it, one per tag set. +StrategyRegistry: TypeAlias = "dict[str, list[TaggedPreRoutingStrategy[Any]]]" + +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( + {"model", "api_base", "api_key", "api_version", "responses_api_version"} +) +_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") +_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 +CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding" +# `_async_get_available_deployment` runs the pre-routing hook only for the strategies with an async +# pick; any other router strategy takes the sync path, which never pre-routes. +_ASYNC_PICK_STRATEGIES: Final = frozenset( + {"usage-based-routing-v2", "simple-shuffle", "cost-based-routing", "latency-based-routing", "least-busy"} +) + + +def picks_async(router_strategy: str | None) -> bool: + """Whether the router's strategy has an async selector: the others pick on the sync path even + for async callers (``_async_get_available_deployment``).""" + return router_strategy in _ASYNC_PICK_STRATEGIES + + +TIER_PARAMS_NEVER_DROPPED: Final = frozenset(all_litellm_params) | frozenset( + { + "additional_drop_params", + "drop_params", + "messages", + "model", + "extra_headers", + "max_tokens", + "max_completion_tokens", + } +) + + +def _json_default_stable_id(value: object) -> str: + return f"{type(value).__module__}.{type(value).__qualname__}" + + +def _deployment_tags(deployment: Deployment) -> tuple[str, ...]: + return tuple(deployment.litellm_params.tags or ()) + + +def _has_registered_strategy(registry: StrategyRegistry, model_name: str, tags: tuple[str, ...]) -> bool: + return any(existing.tags == tags for existing in registry.get(model_name, [])) + + +def _routing_definition_fingerprint(deployment: Deployment) -> str | None: + """Identify configured routing fields, excluding external files and live strategy state.""" + try: + definition: Final = json.dumps( + deployment.litellm_params.model_dump( + include=set(STRATEGY_ROUTER_PARAM_FIELDS | {"model", "tags"}), exclude_none=True, warnings=False + ), + sort_keys=True, + default=_json_default_stable_id, + ) + except (TypeError, ValueError): + return None + return deployment_id(deployment.model_name, {"routing_definition": definition}) + + +def _unregister(registry: StrategyRegistry, model_name: str, tags: tuple[str, ...]) -> bool: + existing: Final = registry.get(model_name, []) + remaining: Final = [entry for entry in existing if entry.tags != tags] + if len(remaining) == len(existing): + return False + if remaining: + registry[model_name] = remaining # rebind-ok: the live registry, edited in place as the Python router does + else: + registry.pop(model_name, None) + return True + + +def is_strategy_marker(deployment: Mapping[str, object]) -> bool: + params: Final = deployment.get("litellm_params") + if not isinstance(params, Mapping): + return False + model: Final = params.get("model") + return isinstance(model, str) and classify_strategy_router_model(model) is not None + + +def _participates_in_adaptive_routing(params: LiteLLM_Params) -> bool: + kind: Final = classify_strategy_router_model(params.model) + if kind == "adaptive": + return True + if kind != "complexity": + return False + config: Final = params.complexity_router_config + return bool(config.get("adaptive")) if config else False + + +class PreRoutingStrategies: + """The pre-routing strategies of one router, kept on its host registry (``registry().plugins`` + reads ``_core_auto_routers`` and its siblings there).""" + + def __init__( + self, hosted: "HostRegistry", host: "PythonHost", ctx: object, capability_limit: object, cache: DualCache + ) -> None: + self._hosted: Final = hosted + self._host: Final = host + self._ctx: Final = ctx + self._capability_limit: object = capability_limit + self._cache: Final = cache + self._affinity: object = None + # The router's own `_ensure_deployment_affinity_callback` (its optional checks), when set. + self.ensure_affinity: Callable[[], None] | None = None + + # -- the registries ------------------------------------------------------------------------ + + @property + def auto(self) -> StrategyRegistry: + return self._hosted._core_auto_routers # pyright: ignore[reportPrivateUsage] # the registries live on the host registry + + @property + def complexity(self) -> StrategyRegistry: + return self._hosted._core_complexity_routers # pyright: ignore[reportPrivateUsage] # see above + + @property + def adaptive(self) -> StrategyRegistry: + return self._hosted._core_adaptive_routers # pyright: ignore[reportPrivateUsage] # see above + + @property + def quality(self) -> StrategyRegistry: + return self._hosted._core_quality_routers # pyright: ignore[reportPrivateUsage] # see above + + def all(self) -> tuple[StrategyRegistry, ...]: + """In the order ``_select_pre_routing_strategy`` searches them.""" + return (self.auto, self.complexity, self.adaptive, self.quality) + + def any_registered(self) -> bool: + return any(self.all()) + + # -- registry hooks (HostRegistry calls these) ----------------------------------------------- + + def reset(self) -> None: + """``set_model_list`` drops the auto, complexity and quality routers (never the adaptive ones).""" + self._hosted._core_quality_routers = {} # pyright: ignore[reportPrivateUsage] # see above + self._hosted._core_complexity_routers = {} # pyright: ignore[reportPrivateUsage] # see above + self._hosted._core_auto_routers = {} # pyright: ignore[reportPrivateUsage] # see above + + def register(self, deployment: Deployment) -> None: + """The end of ``_add_deployment``: an auto, complexity or quality marker registers its router + (adaptive ones wait for ``finalize``).""" + match classify_strategy_router_model(deployment.litellm_params.model): + case "semantic": + self._init_auto_router(deployment) + case "complexity": + self._init_complexity_router(deployment) + case "quality": + self._init_quality_router(deployment) + case _: + pass + + def release(self, deployment: Deployment) -> None: + """``_unregister_pre_routing_strategy_for_deployment``.""" + if not deployment.litellm_params.model.startswith("auto_router/"): + return + model_name: Final = deployment.model_name + tags: Final = _deployment_tags(deployment) + for registry in (self.auto, self.complexity, self.quality): + _ = _unregister(registry, model_name, tags) + if _unregister(self.adaptive, model_name, tags): + self._sync_adaptive_router_hooks() + + def release_entry(self, entry: Mapping[str, object]) -> None: + """``release`` of a deleted model_list entry; a failure only logs, as the delete has happened.""" + try: + self.release(Deployment(**entry)) # pyright: ignore[reportArgumentType] # a model_list entry + except Exception: # noqa: BLE001 # the delete already happened; a failed release only logs + verbose_router_logger.exception( + "delete_deployment: could not release pre-routing strategies for model_id=%s; " + "the deployment is out of the model_list and its indices are repaired", + as_dict(entry.get("model_info")).get("id"), + ) + + def upserted(self, deployment: Deployment, previous: Deployment | None) -> None: + """After an upsert's re-add: rebuild the adaptive routers when it touched one.""" + if _participates_in_adaptive_routing(deployment.litellm_params) or ( + previous is not None and _participates_in_adaptive_routing(previous.litellm_params) + ): + self.finalize() + + def without_capability_limit(self, action: Callable[[], object]) -> None: + """Run ``action`` without judging the capability ceiling: a rollback re-admits a serving router.""" + resolver: Final = self._capability_limit + self._capability_limit = None + try: + _ = action() + finally: + self._capability_limit = resolver + + def finalize(self) -> None: + """``_finalize_adaptive_router_if_configured``: an adaptive router for every adaptive marker, + and the adaptive companion of every complexity router configured with ``adaptive``.""" + for entry in self._hosted._core_model_list or []: # pyright: ignore[reportPrivateUsage] # the host registry's live list + params = entry.get("litellm_params") + lp_model = params.get("model") if isinstance(params, dict) else None + if not (isinstance(lp_model, str) and lp_model.startswith("auto_router/adaptive_router")): + continue + model_name = entry.get("model_name") + if not model_name or not params: + continue + deployment = Deployment( + model_name=str(model_name), + litellm_params=LiteLLM_Params.model_validate(params), + model_info=entry.get("model_info"), # pyright: ignore[reportArgumentType] # the entry's model_info + ) + if _has_registered_strategy(self.adaptive, str(model_name), _deployment_tags(deployment)): + continue + self._init_adaptive_router(deployment) + for model_name, tagged_complexity_routers in self.complexity.items(): + for tagged in tagged_complexity_routers: + complexity_router = tagged.strategy + if not complexity_router.config.adaptive: + continue + if _has_registered_strategy(self.adaptive, model_name, tagged.tags): + continue + adaptive_router = complexity_router._ensure_adaptive_router() + if adaptive_router is not None: + self.adaptive[model_name] = [ + *self.adaptive.get(model_name, []), + TaggedPreRoutingStrategy( + tags=tagged.tags, + strategy=adaptive_router, + definition_fingerprint=tagged.definition_fingerprint, + deployment=tagged.deployment, + ), + ] + self._sync_adaptive_router_hooks() + + def retire(self) -> None: + """Release the adaptive routers' post-call hooks this router registered.""" + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + + own: Final = frozenset(id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive.values())) + hooks: Final[list[object]] = [ # mutable-ok: built here + hook + for hook in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) + if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own + ] + if self._affinity is not None: + hooks.append(self._affinity) + self._affinity = None + if hooks: + self._host.callbacks.drop_callbacks(hooks) + + # -- building -------------------------------------------------------------------------------- + + def _register( + self, registry: StrategyRegistry, deployment: Deployment, strategy: object, strategy_label: str + ) -> None: + tags: Final = _deployment_tags(deployment) + if _has_registered_strategy(registry, deployment.model_name, tags): + raise ValueError( + f"{strategy_label} deployment {deployment.model_name} with tags {list(tags)} already exists. " + "Please use a different model name or set different tags." + ) + registry[deployment.model_name] = [ # rebind-ok: the live registry, edited in place as the Python router does + *registry.get(deployment.model_name, []), + TaggedPreRoutingStrategy( + tags=tags, + strategy=strategy, + definition_fingerprint=_routing_definition_fingerprint(deployment), + deployment=deployment, + ), + ] + + def _capability_violation(self, capability: GatedAutoRouterCapability) -> str | None: + resolver: Final = self._capability_limit + # `self.auto_router_capability_limit() if ... is not None`: a value that cannot be called raises. + limit: Final = resolver() if resolver is not None else None # pyright: ignore[reportCallIssue] # called as given, as Python calls it + others: Final = count_capability_routers( + (entry for entry in self._hosted._core_model_list if isinstance(entry, Mapping)), # pyright: ignore[reportPrivateUsage, reportUnnecessaryIsInstance] # the live list + capability=capability, + ) + return capability_limit_violation(capability=capability, held=others + 1, limit=limit) # pyright: ignore[reportArgumentType] # the resolver's limit + + def _init_auto_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.auto_router.auto_router import AutoRouter + + params: Final = deployment.litellm_params + if params.auto_router_config_path is None and params.auto_router_config is None: + raise ValueError( + "auto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_params" + ) + if params.auto_router_default_model is None: + raise ValueError( + "auto_router_default_model is required for auto-router deployments. Please set it in the litellm_params" + ) + if params.auto_router_embedding_model is None: + raise ValueError( + "auto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params" + ) + max_input_chars: Final = params.auto_router_max_input_chars + router: Final = AutoRouter( + model_name=deployment.model_name, + auto_router_config_path=params.auto_router_config_path, + auto_router_config=params.auto_router_config, + default_model=params.auto_router_default_model, + embedding_model=params.auto_router_embedding_model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + max_input_chars=max_input_chars if max_input_chars is not None else DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, + ) + self._register(self.auto, deployment, router, "Auto-router") + + def _init_complexity_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig + + config: Final = deployment.litellm_params.complexity_router_config + for capability in claimed_capabilities(config): + if (violation := self._capability_violation(capability)) is not None: + raise ValueError(violation) + default_model: Final = ( + ComplexityRouterConfig.model_validate(config).resolve_default_model( + deployment.litellm_params.complexity_router_default_model + ) + if config + else deployment.litellm_params.complexity_router_default_model + ) + if default_model is None: + raise ValueError( + "complexity_router_default_model is required for complexity-router deployments, " + "or configure tiers in complexity_router_config. Please set it in the litellm_params" + ) + router: Final = ComplexityRouter( + model_name=deployment.model_name, + default_model=default_model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + complexity_router_config=config, + ) + self._register(self.complexity, deployment, router, "Complexity-router") + if router._uses_deployment_pin: # pyright: ignore[reportPrivateUsage] # the Python router reads the same flag + self._ensure_deployment_affinity_callback() + + def _ensure_deployment_affinity_callback(self) -> None: + """``_ensure_deployment_affinity_callback``: the affinity check (every global flag off) a + complexity router's session pin needs, registered once.""" + from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck + + if self.ensure_affinity is not None: + self.ensure_affinity() + return + if self._affinity is not None: + return + hosted: Final = self._hosted + + def is_priority_group(model: str) -> bool: + resolved: Final = hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # the host registry's alias lookup + group: Final = hosted.get_routing_group(resolved) + return group is not None and group.routing_strategy == "priority" + + affinity: Final = DeploymentAffinityCheck( + cache=self._cache, + ttl_seconds=3600, + enable_user_key_affinity=False, + enable_responses_api_affinity=False, + enable_session_id_affinity=False, + model_group_affinity_config=None, + is_priority_group=is_priority_group, + session_affinity_repin_on_failover=False, + ) + self._affinity = affinity + self._host.callbacks.add_litellm_callback(affinity) + + def _init_quality_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.quality_router.quality_router import QualityRouter + + config: Final = deployment.litellm_params.quality_router_config + configured: Final = deployment.litellm_params.quality_router_default_model + default_model: Final = configured if configured is not None or not config else config.get("default_model") + if default_model is None: + raise ValueError( + "quality_router_default_model is required for quality-router deployments, " + "or set default_model in quality_router_config. Please configure it in the litellm_params" + ) + router: Final = QualityRouter( + model_name=deployment.model_name, + default_model=default_model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + quality_router_config=config, + ) + self._register(self.quality, deployment, router, "Quality-router") + + def _init_adaptive_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + from litellm.types.router import AdaptiveRouterConfig, AdaptiveRouterPreferences + + raw_config: Final = deployment.litellm_params.adaptive_router_config + if raw_config is None: + raise ValueError("adaptive_router_config is required for adaptive-router deployments.") + config: Final = AdaptiveRouterConfig(**raw_config) + model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} # mutable-ok: built here + model_to_cost: Final[dict[str, float]] = {} # mutable-ok: built here + for name in config.available_models: + deployments = self._hosted._core_deployments_named(name) # pyright: ignore[reportPrivateUsage] # the host registry's name index + if not deployments: + continue + first = deployments[0] + model_info = first.get("model_info") + info: Mapping[str, object] = model_info if isinstance(model_info, dict) else {} + prefs_raw = info.get("adaptive_router_preferences") + if isinstance(prefs_raw, dict): + model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) + cost = _adaptive_cost(as_dict(first.get("litellm_params")), info) + if isinstance(cost, int | float | str): + model_to_cost[name] = float(cost) + router: Final = AdaptiveRouter( + router_name=deployment.model_name, + config=config, + model_to_prefs=model_to_prefs, + model_to_cost=model_to_cost, + ) + self._register(self.adaptive, deployment, router, "Adaptive-router") + self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=router)) + verbose_router_logger.info( + "AdaptiveRouter[%s] initialized with %d models", deployment.model_name, len(config.available_models) + ) + + def _sync_adaptive_router_hooks(self) -> None: + """One post-call hook per registered adaptive router, plus the header hook while any exists.""" + from litellm.router_strategy.adaptive_router.hooks import ( + AdaptiveRouterPostCallHook, + AdaptiveRouterResponseHeaderHook, + ) + + callbacks: Final = self._host.callbacks + for callback in callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): + callbacks.remove_callback_from_all_lists(callback) + for tagged in chain.from_iterable(self.adaptive.values()): + callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)) + if self.adaptive: + callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) + + +# -- the request side -------------------------------------------------------------------------- + + +def _stamp_or_clear_metadata_key( + request_kwargs: dict[str, object], # mutable-ok: the request kwargs, stamped in place + key: str, + value: object | None, +) -> None: + """Write a proxy-internal metadata key for this routing attempt, or clear it from both buckets.""" + if value is None: + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): + if isinstance(bucket, dict): + bucket.pop(key, None) + return + _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) + metadata_bucket[key] = value + + +def _redact_prompt_text_if_needed( + request_kwargs: Mapping[str, object], routing_decision: StandardLoggingRoutingDecision +) -> StandardLoggingRoutingDecision: + from litellm.litellm_core_utils.redact_messages import should_redact_message_logging + + if not should_redact_message_logging( + { + "litellm_params": request_kwargs, + "standard_callback_dynamic_params": request_kwargs.get("standard_callback_dynamic_params"), + } + ): + return routing_decision + kept: Final = without(routing_decision, *PROMPT_QUOTING_ROUTING_DECISION_FIELDS) + return cast(StandardLoggingRoutingDecision, kept) # cast-ok: dropping optional keys preserves the type + + +def _record_routing_decision( + request_kwargs: dict[str, object], # mutable-ok: the request kwargs + routing_decision: StandardLoggingRoutingDecision | None, +) -> None: + """Make the request's metadata describe this routing attempt, and only this one.""" + from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters + from litellm.types.router import BaselineRouteStamp + + phase_attributes(routing_decision_attributes(routing_decision)) + baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None + baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None + router_name: Final = routing_decision.get("router_model_name") if routing_decision else None + caller_parameters: Final = capture_baseline_parameters(request_kwargs) if router_name and baseline_model else None + _stamp_or_clear_metadata_key( + request_kwargs, + "_autorouter_baseline_route", + BaselineRouteStamp(router_name, baseline_model, baseline_id, caller_parameters) + if router_name and baseline_model and baseline_id + else None, + ) + _stamp_or_clear_metadata_key( + request_kwargs, + "routing_decision", + None if routing_decision is None else _redact_prompt_text_if_needed(request_kwargs, routing_decision), + ) + + +def _as_output_cap(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + try: + cap: Final = int(float(value)) + except (ValueError, OverflowError): + return None + return cap if cap >= 0 else None + + +def _stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str, int] | None: + stamped: Final = next( + ( + bucket.get(CLIENT_OUTPUT_CEILING_METADATA_KEY) + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) + if isinstance(bucket, dict) and CLIENT_OUTPUT_CEILING_METADATA_KEY in bucket + ), + None, + ) + if not isinstance(stamped, dict): + return None + return MappingProxyType( + { + str(carrier): cap + for carrier, value in stamped.items() + if carrier in OUTPUT_TOKEN_CEILING_PARAMS and (cap := _as_output_cap(value)) is not None + } + ) + + +def _restore_client_output_ceiling( + request_kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs, restored in place +) -> None: + callers_ceiling: Final = _stamped_client_ceiling(request_kwargs) + if callers_ceiling is None: + return + for carrier in OUTPUT_TOKEN_CEILING_PARAMS: + request_kwargs.pop(carrier, None) + request_kwargs.update(callers_ceiling) + + +def _stamp_client_output_ceiling(request_kwargs: dict[str, object]) -> None: # mutable-ok: the request kwargs + _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) + metadata_bucket.setdefault( + CLIENT_OUTPUT_CEILING_METADATA_KEY, + {carrier: request_kwargs[carrier] for carrier in OUTPUT_TOKEN_CEILING_PARAMS if carrier in request_kwargs}, + ) + + +def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None: # mutable-ok: same + nested: Final = request_kwargs.get(carrier) + if not isinstance(nested, Mapping): + return + sanitized: Final = without(nested, "effort") + if sanitized: + request_kwargs[carrier] = sanitized # rebind-ok: copy-on-write of the request's nested carrier + else: + request_kwargs.pop(carrier, None) + + +def _tier_ceiling_under_the_surface_name( + tier_litellm_params: Mapping[str, object], responses_call: bool +) -> Mapping[str, object]: + surface_key: Final = "max_output_tokens" if responses_call else "max_tokens" + carried: Final = tuple( + key + for key in (surface_key, "max_tokens", "max_completion_tokens", "max_output_tokens") + if key in tier_litellm_params + ) + if not carried: + return tier_litellm_params + return MappingProxyType( + { + **without(tier_litellm_params, *OUTPUT_TOKEN_CEILING_PARAMS), + surface_key: tier_litellm_params[carried[0]], + } + ) + + +def _drop_client_carriers_a_tier_pin_supersedes( + request_kwargs: dict[str, object], # mutable-ok: the request kwargs + tier_litellm_params: Mapping[str, object], +) -> None: + if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_litellm_params): + _stamp_client_output_ceiling(request_kwargs) + for carrier in OUTPUT_TOKEN_CEILING_PARAMS: + request_kwargs.pop(carrier, None) + if "reasoning_effort" not in tier_litellm_params: + return + request_kwargs.pop("thinking", None) + _pop_effort_from_nested_carrier(request_kwargs, "output_config") + _pop_effort_from_nested_carrier(request_kwargs, "reasoning") + + +def _request_header(request_kwargs: Mapping[str, object], header_name: str) -> str | None: + proxy_server_request: Final = request_kwargs.get("proxy_server_request") + if not isinstance(proxy_server_request, Mapping): + return None + headers: Final = proxy_server_request.get("headers") + if not isinstance(headers, Mapping): + return None + return next( + ( + value + for key, value in headers.items() + if isinstance(key, str) and key.lower() == header_name and isinstance(value, str) + ), + None, + ) + + +def _session_router_cache_key(request_kwargs: Mapping[str, object]) -> str | None: + session_id: Final = _request_header(request_kwargs, "x-claude-code-session-id") + if session_id is None or _CLAUDE_CODE_SESSION_ID_RE.fullmatch(session_id) is None: + return None + metadata_name: Final = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata" + metadata: Final = request_kwargs.get(metadata_name) + if not isinstance(metadata, Mapping): + return None + caller_scope: Final = metadata.get("user_api_key_hash") + if not isinstance(caller_scope, str) or not caller_scope: + return None + return f"claude_code_session_router:v1:{caller_scope}:{session_id}" + + +class PreRouter: + """Answers the Rust core's ``pre_route`` steps (``_core_async_pre_routing_hook``).""" + + def __init__( + self, + strategies: PreRoutingStrategies, + hosted: "HostRegistry", + host: "PythonHost", + ctx: object, + routing_cache: DualCache, + ) -> None: + self._strategies: Final = strategies + self._hosted: Final = hosted + self._host: Final = host + self._ctx: Final = ctx + self._session_cache: Final = DualCache(redis_cache=routing_cache.redis_cache, in_memory_cache=InMemoryCache()) + + # -- the step -------------------------------------------------------------------------------- + + async def pre_route( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the request kwargs the hook stamps + *, + messages: object, + input: object, + fallback_depth: int, + router_strategy: str | None, + enable_tag_filtering: bool, + match_any: bool, + public_router: object, + run_plugins: Callable[[str], Awaitable[None]] | None = None, + pass_through: bool = False, + ) -> str | None: + """The start of ``_async_get_available_deployment``: run the hook, record its pick and pin the + tier's litellm_params. The model group to resolve, or ``None`` to resolve ``model`` itself. + ``pass_through`` is ``_async_get_available_deployment_for_pass_through``, which runs the hook + whatever the router's strategy.""" + if not pass_through and not picks_async(router_strategy): + return None + responses_call: Final = input is not None and messages is None + response: Final = await self._hook( + model, + request_kwargs, + messages=messages, + input=input, + fallback_depth=fallback_depth, + enable_tag_filtering=enable_tag_filtering, + match_any=match_any, + public_router=public_router, + run_plugins=run_plugins, + ) + routed: Final = response.model if response is not None else model + if response is not None: + record_pre_routing_selection(request_kwargs, routed) + self._pin_tier_params( + routed, response.litellm_params if response is not None else None, request_kwargs, responses_call + ) + return routed if response is not None else None + + # -- the hook -------------------------------------------------------------------------------- + + async def _hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: see pre_route + *, + messages: object, + input: object, + fallback_depth: int, + enable_tag_filtering: bool, + match_any: bool, + public_router: object, + run_plugins: Callable[[str], Awaitable[None]] | None, + ) -> "PreRoutingHookResponse | None": + _restore_client_output_ceiling(request_kwargs) + _record_routing_decision(request_kwargs, None) + requested: Final = self._hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # the host registry's alias lookup + registered: Final = await self._resolve_session_router( + model, requested, request_kwargs, fallback_depth, enable_tag_filtering, match_any + ) + # The routing plugins run on the session-resolved name, before a strategy is selected, so a + # strategy reads their signals; router-core narrows the pick to what they left. + if run_plugins is not None: + await run_plugins(registered) + selected: Final = self._select_strategy(registered, request_kwargs, enable_tag_filtering, match_any) + if selected is None: + await arm_compaction(request_kwargs, None) + _stamp_or_clear_metadata_key(request_kwargs, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, None) + _stamp_or_clear_metadata_key(request_kwargs, CONSUMED_REQUEST_TAGS_METADATA_KEY, None) + return None + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + + marker: Final = self._selected_marker(registered, selected.tags, request_kwargs) + definition: Final = selected.deployment + config_id: Final = definition.model_info.id if definition else None + updated_at: Final = definition.model_info.updated_at if definition else None + router_kind: Final = classify_strategy_router_model(definition.litellm_params.model) if definition else None + provenance: Final[StandardLoggingRoutingDecision] = { + "router_model_name": definition.model_name if definition else registered, + **({"router_type": router_kind} if router_kind is not None else {}), + **({"router_config_id": config_id} if config_id is not None else {}), + **({"router_config_updated_at": updated_at.isoformat()} if updated_at is not None else {}), + **( + {"router_config_fingerprint": selected.definition_fingerprint} + if selected.definition_fingerprint is not None + else {} + ), + } # pyright: ignore[reportAssignmentType] # the optional keys, as the Python router builds them + phase_attributes(routing_decision_attributes(provenance)) + reject_recursive_compactor(registered) + strategy: Final = selected.strategy + complexity: Final = strategy if isinstance(strategy, ComplexityRouter) else None + await arm_compaction( + request_kwargs, + complexity.config.context_compaction if complexity is not None else None, + _tier_members(complexity.config.tiers) if complexity is not None else (), # pyright: ignore[reportArgumentType] # the tiers' model names + parent_model=model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + allow_escalation=complexity is not None and complexity.config.enable_context_window_escalation, + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + ) + await router_host().authorize_strategy_inference( + deployment=marker, + request_kwargs=request_kwargs, + llm_router=public_router, # pyright: ignore[reportArgumentType] # the litellm.Router this core serves + ) + compression_policy: Final = router_host().compression_policy( + llm_router=public_router, # pyright: ignore[reportArgumentType] # same + model_alias=registered, + request_kwargs=request_kwargs, + request_tags=get_routing_tags_from_request_kwargs(request_kwargs), + ) + independent_compression: Final = compression_policy is not None and not ( + compression_policy.is_same + and compression_policy.model is not None + and router_host().model_hop_compression_armed() + ) + routing_messages: Final = ( + await router_host().messages_for_routing( + policy=compression_policy, + messages=messages, # pyright: ignore[reportArgumentType] # same + request_kwargs=request_kwargs, + ) + if independent_compression + else None + ) + routed: Final = await strategy.async_pre_routing_hook( + model=registered, + request_kwargs=request_kwargs, + messages=routing_messages if routing_messages is not None else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + input=input, # pyright: ignore[reportArgumentType] # the request's input, as given + specific_deployment=False, + ) + hooked: Final[PreRoutingHookResponse | None] = routed + traced: Final = ( + hooked.model_copy(update={"routing_decision": {**provenance, **hooked.routing_decision}}) + if hooked is not None and hooked.routing_decision is not None + else hooked + ) + response: Final = ( + traced.model_copy(update={"messages": messages}) + if traced is not None and routing_messages is not None and traced.messages == routing_messages + else traced + ) + _record_routing_decision(request_kwargs, response.routing_decision if response else None) + _stamp_or_clear_metadata_key( + request_kwargs, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + response.session_affinity_ttl_seconds if response else None, + ) + request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs) + consumed: Final = ( + ConsumedRequestTagsStamp(model_group=response.model, tags=selected.tags) + if response is not None + and selected.tags + and request_tags + and is_valid_deployment_tag(selected.tags, request_tags, match_any) # pyright: ignore[reportArgumentType] # the tags as the Python router passes them + else None + ) + _stamp_or_clear_metadata_key(request_kwargs, CONSUMED_REQUEST_TAGS_METADATA_KEY, consumed) + marker_params: Final = ( + self._forwardable_marker_params(registered, selected.tags, request_kwargs) if response is not None else () + ) + tier_param_keys: Final = tuple(response.litellm_params or ()) if response is not None else () + overridden: Final = frozenset(tier_param_keys) | ( + OUTPUT_TOKEN_CEILING_PARAMS if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_param_keys) else frozenset() + ) + newly_forwarded: Final = tuple( + (key, value) for key, value in marker_params if key not in request_kwargs and key not in overridden + ) + if response is not None: + _stamp_client_output_ceiling(request_kwargs) + request_kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, None) + request_kwargs.update(newly_forwarded) + if newly_forwarded: + request_kwargs.update({_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: tuple(key for key, _ in newly_forwarded)}) + return response + + # -- strategy selection ---------------------------------------------------------------------- + + def _deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[Mapping[str, object]]: + return self._hosted._core_deployments_for_request(model, request_kwargs) # pyright: ignore[reportPrivateUsage] # the host registry's lookup + + def _select_strategy( + self, model: str, request_kwargs: Mapping[str, object], enable_tag_filtering: bool, match_any: bool + ) -> "TaggedPreRoutingStrategy[Any] | None": + """``_select_pre_routing_strategy``.""" + registries: Final = self._strategies.all() + if not any(registries): + return None + deployments: Final = self._deployments_for_request(model, request_kwargs) + registered_names: Final = tuple(dict.fromkeys(str(d.get("model_name")) for d in deployments)) or (model,) + candidates: Final = ( + tuple( # comprehension-ok: registry by registry, name by name, as the Python router searches + tagged for registry in registries for name in registered_names for tagged in registry.get(name, []) + ) + ) + if not candidates: + return None + request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs) + if request_tags: + for tagged in candidates: + if tagged.tags and is_valid_deployment_tag(list(tagged.tags), request_tags, match_any): + return tagged + for tagged in candidates: + if "default" in tagged.tags: + return tagged + request_scoped: Final = request_kwargs.get("enable_tag_filtering") is True + if ( + (enable_tag_filtering or request_scoped) + and all(tagged.tags for tagged in candidates) + and any(not is_strategy_marker(d) for d in deployments) + ): + return None + return candidates[0] + + def _selected_marker( + self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object] + ) -> Mapping[str, object] | None: + markers: Final = tuple( + deployment + for deployment in self._deployments_for_request(model, request_kwargs) + if str(as_dict(deployment.get("litellm_params")).get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + and "model" in as_dict(deployment.get("litellm_params")) + ) + tag_matched: Final = tuple( + deployment + for deployment in markers + if tuple(_tags_of(as_dict(deployment.get("litellm_params")))) == strategy_tags + ) + return tag_matched[0] if tag_matched else (markers[0] if markers else None) + + def _forwardable_marker_params( + self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object] + ) -> tuple[tuple[str, object], ...]: + marker: Final = self._selected_marker(model, strategy_tags, request_kwargs) + if marker is None: + return () + return tuple( + (key, value) + for key, value in as_dict(marker.get("litellm_params")).items() + if key not in _ALIAS_PARAMS_NEVER_FORWARDED + and key not in CustomPricingLiteLLMParams.model_fields + and value is not None + ) + + # -- the Claude Code session binding --------------------------------------------------------- + + @with_service_target(CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET) + async def _delete_binding(self, cache_key: str) -> None: + try: + await self._session_cache.async_delete_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request + verbose_router_logger.warning( + "Failed to delete Claude Code session router binding; the binding may remain until its TTL expires: %s", + e, + ) + + @with_service_target(CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET) + async def _get_binding(self, cache_key: str) -> object: + try: + if self._session_cache.redis_cache is None: + return await self._session_cache.async_get_cache(key=cache_key) + return await self._session_cache.redis_cache.async_get_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # an optional binding must not make routing depend on Redis + log_redis_failure( + verbose_router_logger, + logging.WARNING, + "Failed to read Claude Code session router binding; using the requested model", + e, + ) + return None + + @with_service_target(CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET) + async def _resolve_session_router( + self, + model: str, + registered: str, + request_kwargs: dict[str, object], # mutable-ok: see pre_route + fallback_depth: int, + enable_tag_filtering: bool, + match_any: bool, + ) -> str: + """``_resolve_claude_code_session_router``.""" + if is_native_compaction_call() or not self._strategies.any_registered(): + return registered + cache_key: Final = _session_router_cache_key(request_kwargs) + if cache_key is None: + return registered + depth: Final = fallback_depth or request_kwargs.get("fallback_depth") + if depth not in (None, 0): + return registered + agent_id: Final = _request_header(request_kwargs, "x-claude-code-agent-id") + if agent_id is not None: + bound_model: Final = await self._get_binding(cache_key) + if not isinstance(bound_model, str): + return registered + bound_registered: Final = self._hosted._core_get_model_from_alias(model=bound_model) or bound_model # pyright: ignore[reportPrivateUsage] # see _hook + if self._select_strategy(bound_registered, request_kwargs, enable_tag_filtering, match_any) is None: + await self._delete_binding(cache_key) + return registered + await self._session_cache.async_set_cache( + key=cache_key, value=bound_model, ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS + ) + _stamp_or_clear_metadata_key(request_kwargs, "model_group", bound_model) + return bound_registered + if self._select_strategy(registered, request_kwargs, enable_tag_filtering, match_any) is None: + return registered + await self._session_cache.async_set_cache( + key=cache_key, value=model, ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS + ) + return registered + + # -- tier litellm_params --------------------------------------------------------------------- + + def _tier_params_the_target_accepts( + self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + from litellm.router_rust.invoke.shaping import deployment_accepts_param + + deployments: Final = self._hosted._core_get_model_list(model_name=model) or () # pyright: ignore[reportPrivateUsage] # the host registry's lookup + if not deployments: + return tier_params + allowlisted: Final = _allowlist(tier_params) | _allowlist(request_kwargs) + candidates: Final = provider_rejectable_params(tier_params) - TIER_PARAMS_NEVER_DROPPED - allowlisted + unsupported: Final = frozenset( + param + for param in candidates + if not any(deployment_accepts_param(deployment, model, param, self._host) for deployment in deployments) + ) + if not unsupported: + return tier_params + verbose_router_logger.warning( + "litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them", + ", ".join(sorted(unsupported)), + model, + ) + return MappingProxyType(without(tier_params, *unsupported)) + + def _pin_tier_params( + self, + model: str, + tier_litellm_params: Mapping[str, object] | None, + request_kwargs: dict[str, object], # mutable-ok: see pre_route + responses_call: bool, + ) -> None: + """``_pin_tier_params_onto_request``.""" + if not tier_litellm_params: + return + reject_api_key_secret_reference(tier_litellm_params) + accepted: Final = self._tier_params_the_target_accepts(model, tier_litellm_params, request_kwargs) + surface: Final = _tier_ceiling_under_the_surface_name(accepted, responses_call=responses_call) + _drop_client_carriers_a_tier_pin_supersedes(request_kwargs, surface) + request_kwargs.update(surface) + + +def as_dict(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return {} + return cast(Mapping[str, object], value) # cast-ok: a str-keyed config or request dict + + +def _adaptive_cost(params: Mapping[str, object], info: Mapping[str, object]) -> object: + """A model's input price for the adaptive router: litellm_params win over model_info.""" + cost: Final = params.get("input_cost_per_token") + return cost if cost is not None else info.get("input_cost_per_token") + + +def _tier_members(tiers: Mapping[str, object]) -> tuple[object, ...]: + """Every model the tier pools name, once, in order.""" + pools: Final = ( + (pool,) if isinstance(pool, str) or not isinstance(pool, Iterable) else pool for pool in tiers.values() + ) + return tuple(dict.fromkeys(chain.from_iterable(pools))) + + +def _tags_of(params: Mapping[str, object]) -> Sequence[object]: + tags: Final = params.get("tags") + return tags if isinstance(tags, list | tuple) else () + + +def _allowlist(params: Mapping[str, object]) -> frozenset[str]: + declared: Final = params.get("allowed_openai_params") + if not isinstance(declared, list | tuple | set | frozenset): + return frozenset() + return frozenset(entry for entry in declared if isinstance(entry, str)) + + +__all__ = ["PreRouter", "PreRoutingStrategies", "is_strategy_marker"] diff --git a/litellm/router_rust/steps/selection.py b/litellm/router_rust/steps/selection.py new file mode 100644 index 00000000000..183350bc05f --- /dev/null +++ b/litellm/router_rust/steps/selection.py @@ -0,0 +1,445 @@ +""" +Host-side strategy selectors for ``RustCore``. + +The Rust core decides which strategy picks and whose state it reads (``strategy::routing_context``: +a request override, then the routing group, then the router's own strategy) and asks the host to +run the stateful strategies it does not implement natively through a ``select`` step. The +strategies themselves are the standalone ``litellm/router_strategy`` loggers, which double as +``CustomLogger`` callbacks that record traffic in the host's routing cache. This module builds, +registers and retires them the way the Python router's ``routing_strategy_init``, +``_init_routing_groups`` and ``_get_override_strategy_selector`` do. It never reaches a Python +router core. +""" + +import threading +from collections.abc import Iterable, Mapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy +from litellm.router_strategy.least_busy import ( + CALL_IDENTITY, + SELECTION_RESERVATION_MAX_KEYS, + LeastBusyLoggingHandler, + call_id_of, +) +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler +from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler +from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler +from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 +from litellm.types.router import RoutingGroup, RoutingStrategy + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.caching.in_memory_cache import InMemoryCache + from litellm.router_python_host import PythonHost + +Selector: TypeAlias = ( + LeastBusyLoggingHandler + | LowestCostLoggingHandler + | LowestLatencyLoggingHandler + | LowestTPMLoggingHandler + | LowestTPMLoggingHandler_v2 +) + +# The legacy router attribute that holds the default group's selector, per strategy. +SELECTOR_ATTRIBUTES: Final = MappingProxyType( + { + "least-busy": "leastbusy_logger", + "usage-based-routing": "lowesttpm_logger", + "usage-based-routing-v2": "lowesttpm_logger_v2", + "latency-based-routing": "lowestlatency_logger", + "cost-based-routing": "lowestcost_logger", + } +) + + +def normalize_strategy(strategy: object) -> str | None: + """A strategy as the string Python compares against (``_normalize_strategy``).""" + if isinstance(strategy, RoutingStrategy): + return strategy.value + return strategy if isinstance(strategy, str) else None + + +def build_selector( + strategy: str | None, + args: Mapping[str, object], + cache: "DualCache", + reservations: "InMemoryCache | None" = None, +) -> Selector | None: + """``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every + strategy without a selector. Every least-busy selector of a router keeps its pick + reservations in the router's one ``reservations`` store.""" + routing_args: Final = dict(args) + match strategy: + case "least-busy": + return LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations) + case "usage-based-routing": + return LowestTPMLoggingHandler(router_cache=cache, routing_args=routing_args) + case "usage-based-routing-v2": + return LowestTPMLoggingHandler_v2(router_cache=cache, routing_args=routing_args) + case "latency-based-routing": + return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args) + case "cost-based-routing": + return LowestCostLoggingHandler(router_cache=cache, routing_args={}) + case _: + return None + + +def _callbacks(host: "PythonHost") -> list[object]: # mutable-ok: the process list itself + """``litellm.callbacks``, which the host serves like its other callback lists.""" + return host.callback_list("callbacks") # pyright: ignore[reportArgumentType] # not in the CallbackList literal + + +class StrategySelectors: + """The selectors of one router: the default group's, each routing group's, and the lazily + built per-request override ones.""" + + def __init__(self, host: "PythonHost", cache: "DualCache") -> None: + from litellm.caching.in_memory_cache import InMemoryCache + + self._host: Final = host + self._cache: Final = cache + # `_core_least_busy_reservations`: the picks least-busy reserved, shared by its selectors. + self.reservations: Final = InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS) + self._strategy: str | None = None + # The router's `routing_strategy`, which a direct `routing_strategy_init` leaves as it was. + self._configured: str | None = None + self._default: Selector | None = None + self._groups: dict[str, Selector] = {} # mutable-ok: replaced as routing groups change + self._group_strategies: frozenset[str] = frozenset() + self._overrides: dict[str, Selector | None] = {} # mutable-ok: filled lazily per strategy + self._lock: Final = threading.Lock() + + # -- building ------------------------------------------------------------------------------ + + def _register(self, selector: CustomLogger) -> None: + """``_register_router_selector``: least-busy also counts calls as they start.""" + host: Final = self._host + if isinstance(selector, LeastBusyLoggingHandler): + if isinstance(host.callback_list("input_callback"), list): # pyright: ignore[reportUnnecessaryIsInstance] # a process list a user may have reassigned + host.callbacks.add_litellm_input_callback(selector) + else: + host.set_callback_list("input_callback", [selector]) + if isinstance(_callbacks(host), list): # pyright: ignore[reportUnnecessaryIsInstance] # see above + host.callbacks.add_litellm_callback(selector) + + def _unregister(self, selectors: Sequence[object]) -> None: + live: Final = [selector for selector in selectors if selector is not None] + for selector in live: + if isinstance(selector, BaseRoutingStrategy): + selector.retire() + if live: + self._host.callbacks.drop_callbacks(live) + + def init_default(self, strategy: object, args: Mapping[str, object], *, configured: bool = True) -> None: + """``routing_strategy_init``: drop the old default and override selectors, build the new one. + ``configured``: ``strategy`` is the router's ``routing_strategy`` from now on.""" + self._unregister([self._default, *self._overrides.values()]) + self._overrides = {} + self._default = None + self._strategy = normalize_strategy(strategy) + if configured: + self._configured = self._strategy + self._default = build_selector(self._strategy, args, self._cache, self.reservations) + if self._default is not None: + self._register(self._default) + + def rebuild_default(self, args: Mapping[str, object]) -> None: + """``_apply_updated_routing_strategy_args``: relink the default selector to new args, keeping + the old one when they are invalid.""" + from pydantic import ValidationError + + from litellm._logging import verbose_router_logger + + if self._default is None or self._strategy != self._configured: + return + try: + rebuilt: Final = build_selector(self._strategy, args, self._cache, self.reservations) + except (TypeError, ValidationError): + verbose_router_logger.exception( + "Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy + ) + return + self._unregister([self._default]) + self._default = rebuilt + if rebuilt is not None: + self._register(rebuilt) + + def init_groups(self, groups: Sequence[tuple[str, str, Mapping[str, object]]]) -> None: + """``_replace_routing_groups``: one selector per group, from (name, strategy, args).""" + self._unregister(list(self._groups.values())) + built: Final = { + name: selector + for name, strategy, args in groups + if ( + selector := build_selector( + None if strategy == "priority" else strategy, args, self._cache, self.reservations + ) + ) + is not None + } + for selector in built.values(): + self._register(selector) + self._groups = built + self._group_strategies = frozenset(strategy for _, strategy, _ in groups) + + def retire(self) -> None: + self._unregister([self._default, *self._groups.values(), *self._overrides.values()]) + self.clear() + + def clear(self) -> None: + """Forget every selector, after the caller released them from the host lists.""" + self._default = None + self._groups = {} + self._overrides = {} + + def callback_selectors(self) -> tuple[Selector, ...]: + """``_callback_selectors``: the default group's selector and each routing group's.""" + return tuple(selector for selector in (self._default, *self._groups.values()) if selector is not None) + + def override_selectors(self) -> tuple[Selector, ...]: + """The per-request override selectors built so far.""" + return tuple(selector for selector in self._overrides.values() if selector is not None) + + @property + def default(self) -> Selector | None: + """The default selector of the router's ``routing_strategy``, if it has one.""" + return self._default if self._strategy == self._configured else None + + def attribute(self, name: str) -> Selector | None: + """A legacy ``*_logger`` attribute: the default selector when it is that strategy's.""" + return self._default if SELECTOR_ATTRIBUTES.get(self._strategy or "") == name else None + + def latency_args(self) -> Mapping[str, object] | None: + """The default latency selector's full routing args, as ``get_settings`` reports them.""" + if not isinstance(self._default, LowestLatencyLoggingHandler): + return None + reported: Final[object] = self._default.routing_args.json() # pyright: ignore[reportUnknownMemberType] # RoutingArgs.json is untyped + return as_str_mapping(reported) + + # -- picking ------------------------------------------------------------------------------- + + def selector_for( + self, strategy: str, scope: str, group: str | None, request: Mapping[str, object] | None + ) -> Selector | None: + """The selector a Rust ``select`` step names (``_get_routing_context``).""" + if scope == "group": + return self._groups.get(group or "") + if scope == "override": + return self._override(strategy, request) + return self._default if strategy == self._strategy else None + + def _override(self, strategy: str, request: Mapping[str, object] | None) -> Selector | None: + """``_get_override_strategy_selector`` and ``_bind_override_selector_to_request``.""" + if strategy == self._configured: + return self.default + with self._lock: + if strategy not in self._overrides: + self._overrides[strategy] = build_selector(strategy, {}, self._cache, self.reservations) + selector: Final = self._overrides[strategy] + if selector is not None and request is not None and strategy not in self._registered_strategies(): + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = request.get("litellm_logging_obj") + if isinstance(logging_obj, Logging): + logging_obj.add_dynamic_callback(selector) + return selector + + def _registered_strategies(self) -> frozenset[str]: + return frozenset({*((self._configured,) if self._configured else ()), *self._group_strategies}) + + def is_override(self, strategy: str, selector: object) -> bool: + """Whether ``selector`` is a per-request override one, whose pre-call check the router runs + itself because it is not in the callback lists.""" + return selector is not None and self._overrides.get(strategy) is selector + + +async def pick_async( + strategy: str, + selector: Selector, + model: str, + deployments: list[dict[str, object]], # mutable-ok: the selectors take a list of deployment dicts + messages: object, + input: object, + request_kwargs: dict[str, object] | None, # mutable-ok: the latency selector reads it as given + reserve: bool = False, +) -> object: + """``_select_deployment_async``; ``reserve``: least-busy holds the pick for the request's call.""" + match selector: + case LeastBusyLoggingHandler() if strategy == "least-busy": + return await selector.async_get_available_deployments( + model_group=model, + healthy_deployments=deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), + ) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + case LowestTPMLoggingHandler() if strategy == "usage-based-routing": + return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + input=input, # pyright: ignore[reportArgumentType] # same + ) + case LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in ( + "usage-based-routing-v2", + "cost-based-routing", + ): + return await selector.async_get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + ) + case LowestLatencyLoggingHandler() if strategy == "latency-based-routing": + return await selector.async_get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + request_kwargs=request_kwargs, + ) + case _: + return None + + +def pick_sync( + strategy: str, + selector: Selector, + model: str, + deployments: list[dict[str, object]], # mutable-ok: see pick_async + messages: object, + input: object, + request_kwargs: dict[str, object] | None, # mutable-ok: see pick_async + reserve: bool = False, +) -> object: + """``_select_deployment_sync``: cost-based routing has no sync pick.""" + match selector: + case LeastBusyLoggingHandler() if strategy == "least-busy": + return selector.get_available_deployments( + model_group=model, + healthy_deployments=deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), + ) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in ( + "usage-based-routing", + "usage-based-routing-v2", + ): + return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + ) + case LowestLatencyLoggingHandler() if strategy == "latency-based-routing": + return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + request_kwargs=request_kwargs, + ) + case _: + return None + + +async def routing_pre_call_checks(callbacks: Iterable[object], deployment: Mapping[str, object]) -> None: + """``async_routing_strategy_pre_call_checks``: every ``CustomLogger`` among the router's routing + callbacks (``_routing_callbacks``) checks the picked deployment before the call + (usage-based-routing-v2 counts its rpm here). A check's error fails the attempt; the core + records that failure like any other.""" + for callback in tuple(callbacks): + if isinstance(callback, CustomLogger): + await callback.async_pre_call_check(dict(deployment), None) # pyright: ignore[reportUnknownMemberType] # CustomLogger takes an untyped deployment dict + + +def routing_pre_call_checks_sync(callbacks: Iterable[object], deployment: Mapping[str, object]) -> None: + """``routing_strategy_pre_call_checks``: the sync check of each ``CustomLogger`` callback.""" + for callback in tuple(callbacks): + if isinstance(callback, CustomLogger): + callback.pre_call_check(dict(deployment)) # pyright: ignore[reportUnknownMemberType] # see above + + +def picked_id(picked: object) -> str | None: + """The id of the deployment a selector returned.""" + info: Final = as_str_mapping((as_str_mapping(picked) or {}).get("model_info")) or {} + found: Final = info.get("id") + return str(found) if found is not None else None + + +__all__ = [ + "SELECTOR_ATTRIBUTES", + "Selector", + "StrategySelectors", + "build_selector", + "group_documents", + "group_settings", + "member_map", + "normalize_strategy", + "parse_groups", + "pick_async", + "pick_sync", + "picked_id", + "routing_pre_call_checks", + "routing_pre_call_checks_sync", +] + + +def parse_groups( + groups_input: object, model_names: frozenset[str], aliases: frozenset[str] +) -> tuple[RoutingGroup, ...]: + """``_init_routing_groups``' validation: ``parse_routing_groups``, then a priority group may not + share its name with a model group or alias (a plain group only warns: its name is shadowed).""" + from litellm._logging import verbose_router_logger + from litellm.litellm_core_utils.routing_groups import ( + parse_routing_groups, # pyright: ignore[reportUnknownVariableType] # takes untyped dicts + ) + + if not groups_input or not isinstance(groups_input, list | tuple): + return () + groups: Final = parse_routing_groups( # pyright: ignore[reportUnknownVariableType] # returns RoutingGroup models + list(groups_input), # pyright: ignore[reportUnknownArgumentType] # validated by parse_routing_groups + known_model_names=model_names, + ) + for group in groups: + if group.group_name in model_names or group.group_name in aliases: + if group.routing_strategy == "priority": + raise ValueError("Priority routing group names must not shadow a model or alias") + verbose_router_logger.warning( + "routing_groups: group_name '%s' is shadowed by an existing model_name or model_group_alias; " + "the group's strategy still applies to its members, but the name is not callable until renamed.", + group.group_name, + ) + return groups + + +def group_documents(groups: Sequence[RoutingGroup]) -> list[Mapping[str, object]]: # mutable-ok: a JSON list + """The routing groups as the Rust core reads them.""" + return [ + { + "group_name": group.group_name, + "models": list(group.models), + "routing_strategy": group.routing_strategy, + **({"model_priorities": dict(group.model_priorities)} if group.model_priorities is not None else {}), + } + for group in groups + ] + + +def group_settings(groups: Sequence[RoutingGroup]) -> list[object]: # mutable-ok: the settings snapshot's list + """``get_settings()['routing_groups']``.""" + return [ + group.model_dump(exclude={"model_priorities"}) if group.model_priorities is None else group.model_dump() + for group in groups + ] + + +def member_map(groups: Sequence[RoutingGroup]) -> Mapping[str, str]: + """``_model_to_group``: each member model group of a non-priority routing group, to the group + (a model listed by two groups maps to the last).""" + kept: Final = [group for group in groups if group.routing_strategy != "priority"] + return dict(chain.from_iterable(dict.fromkeys(group.models, group.group_name).items() for group in kept)) diff --git a/litellm/router_rust/surface/README.md b/litellm/router_rust/surface/README.md new file mode 100644 index 00000000000..ad9c0c62ed3 --- /dev/null +++ b/litellm/router_rust/surface/README.md @@ -0,0 +1,14 @@ +# surface + +The Python `Router` API, kept over router-core's state so Python callers see today's behavior. router-core holds the state and the rules; these modules are views and adapters. + +- `registry`, `model_list`, `indexes`, `members`: the deployment list, indexes and lookups; writes become changesets +- `legacy_members`: the old public members outside the entry points +- `model_info`: model and group info, answering the `model_info` step's lookups +- `replay`: re-registers pricing after a price data reload +- `settings`: `update_settings` rules and the settings view +- `health`, `health_routing`, `retries`: outcome callbacks, health-check routing and retry answers +- `fallbacks`: fallback config validation +- `alerting`: Slack alerting + +A Rust host needs none of this. diff --git a/litellm/router_rust/surface/__init__.py b/litellm/router_rust/surface/__init__.py new file mode 100644 index 00000000000..74704783ecb --- /dev/null +++ b/litellm/router_rust/surface/__init__.py @@ -0,0 +1 @@ +"""The Python ``Router`` API kept over router-core's state, for Python callers.""" diff --git a/litellm/router_rust/surface/alerting.py b/litellm/router_rust/surface/alerting.py new file mode 100644 index 00000000000..760eecb35f4 --- /dev/null +++ b/litellm/router_rust/surface/alerting.py @@ -0,0 +1,33 @@ +"""The Rust core's Slack alerting: ``alerting_config`` built into a ``SlackAlerting`` logger on the +host, as the Python core's ``_initialize_alerting`` builds it.""" + +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.router_python_host import PythonHost + from litellm.types.router import AlertingConfig + + +def initialize_alerting(host: "PythonHost", alerting_config: "AlertingConfig | None") -> "SlackAlerting | None": + """``_initialize_alerting``: a ``SlackAlerting`` logger with the config's threshold and webhook, + added to the host's callbacks, and its slow-response check to the host's success callbacks. + No config builds nothing.""" + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + + if alerting_config is None: + return None + logger: Final = SlackAlerting( + alerting_threshold=alerting_config.alerting_threshold, + alerting=["slack"], + default_webhook_url=alerting_config.webhook_url, + ) + host.callbacks.add_litellm_callback(logger) + host.callbacks.add_litellm_success_callback(logger.response_taking_too_long_callback) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # SlackAlerting's untyped callback + verbose_router_logger.info("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") + return logger + + +__all__ = ["initialize_alerting"] diff --git a/litellm/router_rust/surface/fallbacks.py b/litellm/router_rust/surface/fallbacks.py new file mode 100644 index 00000000000..48fbc972b43 --- /dev/null +++ b/litellm/router_rust/surface/fallbacks.py @@ -0,0 +1,129 @@ +""" +Fallback configuration checks ``RustCore`` runs on the host: ``validate_fallbacks``, with the +Python router's error texts. router-core walks the fallbacks; this module only refuses a list it +could not walk the way the Python router would, before the core is built. +""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +from litellm.router_utils.fallback_event_handlers import check_non_standard_fallback_format + + +def refuse_malformed_fallbacks(fallback_param: Sequence[object] | None) -> None: + """``Router.validate_fallbacks``: every entry is a one-key dict whose targets are a list (or a + client-side dict target).""" + if fallback_param is None: + return + for fallback_dict in fallback_param: + if not isinstance(fallback_dict, dict): + raise ValueError(f"Item '{fallback_dict}' is not a dictionary.") # noqa: TRY004 # the Python router's error class + if len(fallback_dict) != 1: # pyright: ignore[reportUnknownArgumentType] # only the size is read + raise ValueError( + f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys." # pyright: ignore[reportUnknownArgumentType] # see above + ) + ((model_group, targets),) = fallback_dict.items() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # an entry as configured + entry: list[object] = [fallback_dict] # mutable-ok: the one-entry list the check reads + if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=entry): + continue + raise TypeError( + f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. " + f"Use {{'{model_group}': ['{targets}']}}" + ) + + +def rejected_fallback_target(targets: Sequence[object]) -> Exception | None: + """The error ``run_async_fallback`` raises before its walk when a chain, from any list and of + any level, has a dict target a request may not set (a server-owned federation field, an + ``oidc/`` api key): every target of that walk is refused.""" + from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference + from litellm.types.router import reject_server_owned_wif_params + + for target in targets: + if isinstance(target, dict): + try: + reject_server_owned_wif_params(target) # pyright: ignore[reportUnknownArgumentType] # a chain as configured + reject_api_key_secret_reference(target) # pyright: ignore[reportUnknownArgumentType] # see above + except Exception as error: # noqa: BLE001 # the walk's own refusal, reported as its failure + return error + return None + + +# The `mock_testing_*` flags the router reads, and the mock failure each asks router-core for. +MOCK_FLAGS: Final = MappingProxyType( + { + "mock_testing_fallbacks": "fallbacks", + "mock_testing_context_fallbacks": "context_window", + "mock_testing_content_policy_fallbacks": "content_policy", + "mock_testing_rate_limit_error": "rate_limit", + } +) + + +def mock_failure(request: Mapping[str, object]) -> str | None: + """The mock failure a request asks for (``MockRouterTestingParams.from_kwargs``, then + ``_handle_mock_testing_rate_limit_error``): the first fallback flag that is ``True`` (a string + is read as a bool), else the rate limit flag when it is ``True`` itself.""" + from litellm.secret_managers.main import str_to_bool + + for flag in ("mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks"): + value = request.get(flag) + if (str_to_bool(value) if isinstance(value, str) else value) is True: + return MOCK_FLAGS[flag] + return "rate_limit" if request.get("mock_testing_rate_limit_error") is True else None + + +def mock_error( + mock: str, model: str, num_retries: object, request: Mapping[str, object], configured: Mapping[str, object] +) -> Exception: + """The exception the Python router raises for a mock failure; ``configured`` holds the + router's fallback lists, which the request's own replace.""" + import litellm.exceptions as litellm_exceptions + + def listed(name: str) -> object: + return request.get(name, configured.get(name)) + + if mock == "rate_limit": + return litellm_exceptions.RateLimitError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a rate limit error.", + num_retries=num_retries if isinstance(num_retries, int) else None, + ) + if mock == "context_window": + return litellm_exceptions.ContextWindowExceededError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a fallback. \ + Context_Window_Fallbacks={listed('context_window_fallbacks')}", + ) + if mock == "content_policy": + return litellm_exceptions.ContentPolicyViolationError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a fallback. \ + Context_Policy_Fallbacks={listed('content_policy_fallbacks')}", + ) + return litellm_exceptions.InternalServerError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a fallback. Fallbacks={listed('fallbacks')}", + ) + + +class RustFallbacks: + """Fallback members served on the host.""" + + def validate_fallbacks(self, fallback_param: Sequence[object] | None) -> None: + refuse_malformed_fallbacks(fallback_param) + + +__all__ = [ + "MOCK_FLAGS", + "RustFallbacks", + "mock_error", + "mock_failure", + "refuse_malformed_fallbacks", + "rejected_fallback_target", +] diff --git a/litellm/router_rust/surface/health.py b/litellm/router_rust/surface/health.py new file mode 100644 index 00000000000..5547471855a --- /dev/null +++ b/litellm/router_rust/surface/health.py @@ -0,0 +1,558 @@ +""" +The deployment outcome callbacks ``RustCore`` serves as members: ``deployment_callback_on_failure``, +``deployment_callback_on_success`` and ``sync_deployment_callback_on_success``. router-core counts +the outcome and decides the cooldown; this module reads litellm's logging kwargs, drops the +failures the Python router does not hold against the deployment, and drives the Rust call. The +stream wrappers report a stream's outcome through the same members when the stream ends. +""" + +from collections.abc import Mapping +from datetime import datetime +from typing import TYPE_CHECKING, Final, Literal + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +if TYPE_CHECKING: + from litellm.router_rust.steps.callbacks import OptionalChecks + from litellm.router_rust.surface.registry import HostRegistry + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.rust_bridge._native import ( + RouterCall, + RouterCoreHandle, + _FailureReportSteps, # pyright: ignore[reportPrivateUsage] # the stub's step shapes + _SuccessReportSteps, # pyright: ignore[reportPrivateUsage] # see above + ) + from litellm.types.router import Deployment + + +def _cast_status(exception_status: object) -> int | None: + if isinstance(exception_status, int) and not isinstance(exception_status, bool): + return exception_status + if isinstance(exception_status, str): + try: + return int(exception_status) + except ValueError: + return 500 + return None + + +def failure_status(exception_status: object) -> int | None: + """``cast_exception_status_to_int``: a number as given, a numeric string read as one, any + other string 500. A number no status can hold reads as no status.""" + status: Final = _cast_status(exception_status) + return status if status is not None and 0 <= status <= 65535 else None + + +def _logged_deployment_id(kwargs: Mapping[str, object]) -> object: + """``litellm_params.model_info.id`` of a logged call.""" + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + return (as_str_mapping(litellm_params.get("model_info")) or {}).get("id") + + +def first_cooldown(*sources: Mapping[str, object] | None) -> int | float | None: + """``_first_present(*sources, key="cooldown_time")``, the number as it was written.""" + for source in sources: + value = source.get("cooldown_time") if source is not None else None + if isinstance(value, int | float) and not isinstance(value, bool): + return value + return None + + +def _configured_cooldown(model_info: object, litellm_params: Mapping[str, object]) -> float | None: + """``first_cooldown`` of a logged call, as the float router-core takes.""" + seconds: Final = first_cooldown(as_str_mapping(model_info), litellm_params) + return None if seconds is None else float(seconds) + + +def python_cooldown_time(seconds: float, *sources: float | None) -> float: + """``seconds`` as ``deployment_callback_on_failure`` hands them on. Router-core's duration is + always a float; Python's is the number that won its precedence, an int or a float as it was + written. ``sources`` run in that precedence, and only the type is read from them.""" + return next((source for source in sources if source is not None and source == seconds), seconds) + + +def _fallback_cooldown(deployment: "Deployment | None") -> float | None: + """``first_cooldown`` of the deployment a failed hop tried, as ``_core_get_model_info`` lists it.""" + if deployment is None: + return None + listed: Final = deployment.model_dump() + return first_cooldown(as_str_mapping(listed.get("model_info")), as_str_mapping(listed.get("litellm_params"))) + + +def _deployment_cooldown(deployment: Mapping[str, object]) -> float | None: + """``_configured_cooldown`` of a deployment as listed.""" + return _configured_cooldown(deployment.get("model_info"), as_str_mapping(deployment.get("litellm_params")) or {}) + + +def logged_call(op: str, kwargs: Mapping[str, object], started: datetime | None = None) -> Mapping[str, object]: + """The logging kwargs ``deployment_callback_on_failure`` would read for a call made with + ``kwargs``: the call's logging object's (a proxy request carries one), else what the request says. + Read before the failure logger stamps the call's ``end_time``, so it is stamped here.""" + ended: Final = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline's api_call_start_time + details: Final = as_str_mapping(getattr(kwargs.get("litellm_logging_obj"), "model_call_details", None)) + call_type: Final = op.rsplit(".", 1)[-1] + if details is not None: + return {"call_type": call_type, **details, "end_time": ended} + fields: Final = ("metadata", "litellm_metadata", "client_side_timeout", "timeout") + return { + "call_type": call_type, + "litellm_params": {name: kwargs.get(name) for name in fields}, + "start_time": started, + "end_time": ended, + } + + +def _authenticates_per_caller(litellm_params: object) -> bool: + """``github_copilot_per_user_credential_name`` in ``_should_run_cooldown_logic``. Per-user params + with no credential raise there, which ends Python's failure callback short of any cooldown.""" + from litellm.exceptions import BadRequestError + from litellm.llms.github_copilot.per_user_auth import github_copilot_per_user_credential_name + + try: + return github_copilot_per_user_credential_name(litellm_params) is not None + except BadRequestError: + return True + + +def failure_already_logged(kwargs: Mapping[str, object], event: Literal["async_failure", "sync_failure"]) -> bool: + """``should_run_logging``: a logging object logs one failure a request. The attempts of a request + that shares one (``litellm_logging_obj``) after the first logged failure run no failure callback; + without one, every litellm call logs on its own.""" + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + return isinstance(logging_obj, Logging) and logging_obj.model_call_details.get(f"has_logged_{event}") is True + + +def _success_target(kwargs: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None: + """The deployment id and metadata a success callback counts under, or None.""" + params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + metadata: Final = as_str_mapping(params.get("metadata")) + if metadata is None: + return None + model_info: Final = as_str_mapping(params.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + if metadata.get("model_group") is None or deployment_id is None: + return None + return str(deployment_id), metadata + + +class RustHealthCallbacks: + """Outcome callbacks read into the Rust core's health state.""" + + _handle: "RouterCoreHandle" + _cooldown_given: object + + def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]: + raise NotImplementedError + + async def _drive_cache_steps( + self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]" + ) -> Mapping[str, object]: + raise NotImplementedError + + def _sync_allowed_fails(self) -> None: + raise NotImplementedError + + def _core_get_allowed_fails_from_policy(self, exception: Exception) -> int | None: + """The router ``allowed_fails_policy``'s allowance for this exception, decided by router-core.""" + from litellm.router_rust.core import attempt_failure + + return self._handle.allowed_fails_for(dict(attempt_failure(exception))) + + def _deployment_of(self, deployment_id: object) -> "Deployment | None": + # RustCore sets `_hosted` as an attribute in its constructor. + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # RustCore's own attribute + return hosted._core_get_deployment(model_id=deployment_id) if isinstance(deployment_id, str) else None # pyright: ignore[reportPrivateUsage] # the host registry's id lookup + + def deployment_tracks_usage(self, deployment_id: object, model_group: object) -> bool: + """``_cached_deployment_tracks_usage``. A deployment whose model is not mapped raises there, + which ends Python's usage write, so it reads as not tracked.""" + # RustCore sets `_hosted` as an attribute in its constructor. + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # RustCore's own attribute + if not isinstance(deployment_id, str) or not isinstance(model_group, str): + return False + try: + return hosted.cached_deployment_tracks_usage(deployment_id, model_group) + except Exception: # noqa: BLE001 # as `deployment_callback_on_success` swallows it + return False + + def cooldown_event(self, deployment_id: str, exception_status: object, cooldown_time: float | None) -> None: + """The end of ``set_cooldown_deployments``, once the cooldown is written: the cooldown event + Prometheus reads, as a task of the running loop. Outside one this raises ``RuntimeError``.""" + import asyncio + + from litellm.router_api import RoutingContext + from litellm.router_utils import cooldown_handlers + + _ = asyncio.get_running_loop() + _ = asyncio.create_task( # fire and forget, as set_cooldown_deployments schedules it + cooldown_handlers.router_cooldown_event_callback( + ctx=RoutingContext(self), # pyright: ignore[reportArgumentType] # the core's RoutingContext + deployment_id=deployment_id, + exception_status=exception_status, # pyright: ignore[reportArgumentType] # the status as raised + cooldown_time=cooldown_time, + ) + ) + + def cooldown_event_on_own_loop( + self, deployment_id: str, exception_status: object, cooldown_time: float | None + ) -> None: + """``cooldown_event`` for a sync walk ``function_with_fallbacks`` would run on its private + loop: ``set_cooldown_deployments`` finds that loop however the caller runs, and the event's + task keeps what it raises to itself.""" + import asyncio + + from litellm.litellm_core_utils import asyncify + + async def fire() -> None: + self.cooldown_event(deployment_id, exception_status, cooldown_time) + await asyncio.sleep(0) + + asyncify.run_async_function(fire) # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + + def python_cooldown(self, seconds: float, configured: float | None = None, error: object = None) -> float: + """The ``time_to_cooldown`` Python hands ``set_cooldown_deployments``, for router-core's + ``seconds``. A failure callback's precedence is the ``configured`` number, then the ``retry-after`` + header of ``error``, then the router's own; a path passing neither hands on the router's own.""" + from litellm.router_rust.core import retry_after_seconds + + retry_after: Final = retry_after_seconds(error) if isinstance(error, BaseException) else None + default: Final = self._cooldown_given + return python_cooldown_time( + seconds, + configured, + None if retry_after is None else int(retry_after), + default if isinstance(default, int | float) else None, + ) + + def cooled_down( + self, + done: Mapping[str, object], + deployment_id: object, + exception_status: object, + *, + own_loop: bool = False, + configured: float | None = None, + error: object = None, + ) -> bool: + """Whether a failure report cooled the deployment down, with its cooldown event, which + carries ``python_cooldown`` of ``configured`` and ``error``.""" + cooled: Final = bool(done.get("cooled")) + if cooled and isinstance(deployment_id, str): + seconds: Final = done.get("cooldown_time") + event: Final = self.cooldown_event_on_own_loop if own_loop else self.cooldown_event + event( + deployment_id, + exception_status, + self.python_cooldown(seconds, configured, error) if isinstance(seconds, float) else None, + ) + return cooled + + def per_user_deployment(self, deployment_id: object) -> bool: + """The failed deployment authenticates per caller (``_should_run_cooldown_logic``), which is + all ``set_cooldown_deployments`` reads when it is called without the request's kwargs.""" + deployment: Final = self._deployment_of(deployment_id) + return deployment is not None and _authenticates_per_caller(deployment.litellm_params) + + def failure_exits( + self, exception: object, logged: Mapping[str, object], deployment_id: object + ) -> Mapping[str, bool]: + """What the Python router does not hold against the deployment, for every failure it logs. + ``outside_deployment``: ``deployment_callback_on_failure``'s early exits, counting nothing. + ``caller_scoped``: ``_should_run_cooldown_logic``'s per-caller exits, counted but not cooled; + its exits on the exception's class are router-core's (``FailureKind``).""" + from litellm.router_utils.cooldown_handlers import ( + _is_per_user_provider_request, # pyright: ignore[reportPrivateUsage] # the cooldown check's own rule + is_background_response_poll_not_found, + is_caller_scoped_auth_failure, + is_caller_timeout_408, + is_failure_outside_selected_deployment, + is_video_lookup_not_found, + ) + + status: Final[object] = getattr(exception, "status_code", "") + litellm_params: Final = as_str_mapping(logged.get("litellm_params")) or {} + outside: Final = ( + is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None) + or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params)) + or is_video_lookup_not_found(exception, logged) + or is_caller_timeout_408(logged, status) # pyright: ignore[reportArgumentType] # the status as raised + or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status) # pyright: ignore[reportArgumentType] # see above + ) + return { + "outside_deployment": outside, + "caller_scoped": _is_per_user_provider_request(logged) or self.per_user_deployment(deployment_id), + } + + def trigger_fallback_cooldown( + self, kwargs: Mapping[str, object], exception: object, generic: bool, *, own_loop: bool = False + ) -> None: + """The failed fallback hop's ``_trigger_cooldown_for_failed_deployment``, which runs only when + the request's logging object blocked the hop's failure callbacks. ``generic``: the call's kwargs + carry ``original_generic_function``. ``own_loop``: see ``cooldown_event_on_own_loop``.""" + from litellm._logging import verbose_router_logger + from litellm.router_utils.fallback_event_handlers import ( + _failure_callbacks_blocked_logging_object, # pyright: ignore[reportPrivateUsage] # the fallback path's own rule + ) + + blocked: Final = _failure_callbacks_blocked_logging_object(kwargs) + if blocked is None or not isinstance(exception, Exception): + return + try: + call: Final = self._fallback_failure_call(kwargs, exception, blocked.model_call_details, generic) + if call is not None: + _ = self.cooled_down( + self._drive_sync_steps(call), + getattr(exception, "failed_deployment_id", None), + getattr(exception, "status_code", ""), + own_loop=own_loop, + configured=_fallback_cooldown( + self._deployment_of(getattr(exception, "failed_deployment_id", None)) + ), + error=exception, + ) + except Exception as e: # noqa: BLE001 # best effort, as the Python helper: it never breaks the fallback response + verbose_router_logger.debug("Error triggering cooldown for fallback deployment: %s", e) + + def _fallback_failure_call( + self, + kwargs: Mapping[str, object], + exception: Exception, + model_call_details: Mapping[str, object], + generic: bool, + ) -> "RouterCall[_FailureReportSteps] | None": + """The Rust call recording a failed hop, or None past the exits of + ``_trigger_cooldown_for_failed_deployment``: another list than ``failure_exits``, checked per + failed hop. The selection tag is not on it; a stream stall and a generic call's 404 are.""" + from litellm.router_rust.core import attempt_failure + from litellm.router_utils.cooldown_handlers import ( + _is_per_user_provider_request, # pyright: ignore[reportPrivateUsage] # the cooldown check's own rule + cast_exception_status_to_int, + is_advisor_orchestration_failure, + is_caller_scoped_auth_failure, + is_caller_timeout_408, + is_stream_stall, + ) + from litellm.router_utils.fallback_event_handlers import ( + _REQUEST_SCOPED_STATUS_CODES, # pyright: ignore[reportPrivateUsage] # the fallback path's own rule + ) + + self._sync_allowed_fails() + exception_status: Final[object] = getattr(exception, "status_code", "") + if ( + is_advisor_orchestration_failure(exception) + or is_stream_stall(exception) + or (generic and cast_exception_status_to_int(exception_status) in _REQUEST_SCOPED_STATUS_CODES) # pyright: ignore[reportArgumentType] # the status as raised + or is_caller_timeout_408(model_call_details, exception_status, ended=datetime.now()) # pyright: ignore[reportArgumentType] # noqa: DTZ005 # see logged_call + ): + return None + deployment_id: Final[object] = getattr(exception, "failed_deployment_id", None) + if not isinstance(deployment_id, str): + return None + deployment: Final = self._deployment_of(deployment_id) + if is_caller_scoped_auth_failure(deployment, exception_status): # pyright: ignore[reportArgumentType] # see above + return None + return self._handle.report_failure( + deployment_id, + { + **attempt_failure(exception), + "status": failure_status(exception_status), + "caller_scoped": _is_per_user_provider_request(kwargs) or self.per_user_deployment(deployment_id), + }, + deployment_cooldown=True, + count_only=exception_status == "", + cooldown_time=None if deployment is None else _deployment_cooldown(deployment.model_dump()), + ) + + def deployment_callback_on_failure( + self, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> bool: + """Count the failure and cool the deployment down when the rules say so.""" + call: Final = self._failure_call(kwargs) + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + return call is not None and self.cooled_down( + self._drive_sync_steps(call), + _logged_deployment_id(kwargs), + getattr(kwargs.get("exception"), "status_code", ""), + configured=first_cooldown(as_str_mapping(litellm_params.get("model_info")), litellm_params), + error=kwargs.get("exception"), + ) + + async def report_stream_failure( + self, kwargs: Mapping[str, object], callbacks: Mapping[str, bool] | None = None + ) -> None: + """A stream's failure after its attempt returned it, as an async call's failure callbacks + record it: counted, cooled down by the rules, and one more request this minute + (``async_deployment_callback_on_failure``). No cooldown event: the stream logs its failure + through ``dispatch_failure_handlers``, whose sync callbacks run in an executor thread, where + ``set_cooldown_deployments`` has no loop to schedule one on.""" + call: Final = self._failure_call(kwargs, counts_request=True, callbacks=callbacks) + if call is not None: + _ = await self._drive_cache_steps(call) + + def report_stream_failure_sync( + self, kwargs: Mapping[str, object], callbacks: Mapping[str, bool] | None = None + ) -> None: + """``report_stream_failure`` for a sync stream: a sync call's failure callback counts no request. + No cooldown event: the stream logs its failure in a ``threading.Thread`` of its own, which + has no running loop whatever the consumer's thread has.""" + call: Final = self._failure_call(kwargs, callbacks=callbacks) + if call is not None: + _ = self._drive_sync_steps(call) + + def _failure_call( + self, + kwargs: Mapping[str, object], + counts_request: bool = False, + callbacks: Mapping[str, bool] | None = None, + ) -> "RouterCall[_FailureReportSteps] | None": + """The Rust call recording a logged failure, or None when it records nothing: one the Python + router does not hold against the deployment is still a request of the minute + (``async_deployment_callback_on_failure`` makes none of the failure callback's exits).""" + from litellm.litellm_core_utils import core_helpers + from litellm.router_rust.core import attempt_failure + + self._sync_allowed_fails() + exception: Final = kwargs.get("exception") + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + model_info: Final = litellm_params.get("model_info", {}) + info: Final = as_str_mapping(model_info) + deployment_id: Final = info.get("id") if info is not None else None + if deployment_id is None: + return None + exits: Final = self.failure_exits(exception, kwargs, deployment_id) + if exits["outside_deployment"] and not counts_request: + return None + exception_status: Final[object] = getattr(exception, "status_code", "") + described: Final[Mapping[str, object]] = ( + attempt_failure(exception) if isinstance(exception, BaseException) else {"ok": False, "kind": "APIError"} + ) + failure: Final = {**described, **exits, "status": failure_status(exception_status)} + metadata: Final[object] = core_helpers.get_litellm_metadata_from_kwargs(dict(kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # an untyped helper's dict + group: Final = (as_str_mapping(metadata) or {}).get("model_group") # pyright: ignore[reportUnknownArgumentType] # read as a mapping + usage_model: Final = (as_str_mapping(metadata) or {}).get("deployment") # pyright: ignore[reportUnknownArgumentType] # see above + return self._handle.report_failure( + str(deployment_id), + failure, + deployment_cooldown=True, + count_only=exception_status == "", + requested_group=group if isinstance(group, str) else None, + cooldown_time=_configured_cooldown(model_info, litellm_params), + counts_request=counts_request, + usage_model=usage_model if isinstance(usage_model, str) else None, + callbacks=callbacks, + ) + + def sync_deployment_callback_on_success( + self, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> str | None: + """Count a success this minute; returns the counter's key.""" + return self.report_stream_success_sync(kwargs) + + def report_stream_success_sync( + self, kwargs: Mapping[str, object], callbacks: Mapping[str, bool] | None = None + ) -> str | None: + """``sync_deployment_callback_on_success``, for a sync stream that ended.""" + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + + if is_batch_retrieve_call_type(kwargs.get("call_type")): + return None + target: Final = _success_target(kwargs) + if target is None: + return None + key: Final = self._drive_sync_steps(self._handle.count_success(target[0], callbacks)).get("key") + return key if isinstance(key, str) else None + + async def repin_after_success(self, kwargs: Mapping[str, object], completion_response: object) -> None: + """The session re-pin ``deployment_callback_on_success`` runs for a success logged against a + deployment the router serves (``_repin_session_after_success``). The Rust core runs it from the + attempt itself, or from the stream wrapper when a stream ends.""" + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + + call_type: Final = kwargs.get("call_type") + if call_type == "_arealtime" or is_batch_retrieve_call_type(call_type): + return + target: Final = _success_target(kwargs) + # RustCore sets these as attributes in its constructor. + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # RustCore's own attribute + if target is None or hosted._core_get_deployment(model_id=target[0]) is None: # pyright: ignore[reportPrivateUsage] # the host registry's id lookup + return + checks: Final[OptionalChecks] = self.__dict__["_checks"] # pyright: ignore[reportAny] # see above + cooldowns: Final[CooldownCache] = self.__dict__["_cooldowns"] # pyright: ignore[reportAny] # see above + await checks.repin_session_after_success( + as_str_mapping(kwargs.get("litellm_params")) or {}, completion_response, cooldowns + ) + + async def deployment_callback_on_success( + self, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> None: + """Count a success and the call's tokens and request against a tracked deployment.""" + await self.report_stream_success(kwargs, completion_response) + + async def report_stream_success( + self, + kwargs: Mapping[str, object], + completion_response: object, + callbacks: Mapping[str, bool] | None = None, + ) -> None: + """``deployment_callback_on_success``, for a stream that ended. The re-pin is the host's half + of that callback, so it reads ``callbacks`` here.""" + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens + + call_type: Final = kwargs.get("call_type") + if call_type == "_arealtime" or is_batch_retrieve_call_type(call_type): + return + logged: Final = as_str_mapping(kwargs.get("standard_logging_object")) + if logged is None: + return + target: Final = _success_target(kwargs) + if target is None: + return + if (callbacks or {}).get("success", True): + await self.repin_after_success(kwargs, completion_response) + tokens: Final[object] = logged.get("total_tokens", 0) + total: Final = float(tokens) if isinstance(tokens, int | float) else 0.0 + counted: Final = get_counted_usage_tokens(as_str_mapping(kwargs.get("litellm_params")) or {}) + usage_model: Final = target[1].get("deployment") + await self._drive_cache_steps( + self._handle.report_success( + target[0], + total if counted is None else max(0.0, total - counted), + 1.0 if counted is None else 0.0, + usage_model if isinstance(usage_model, str) else None, + self.deployment_tracks_usage(target[0], target[1].get("model_group")), + callbacks, + ) + ) + + async def count_realtime_session_success(self, deployment_id: str) -> None: + """``increment_deployment_successes_for_current_minute`` for a realtime session that ended + cleanly: a success alone, as ``deployment_callback_on_success`` skips ``_arealtime``. A + counter that cannot be written does not fail the served session.""" + from litellm._logging import verbose_router_logger + + try: + await self._drive_cache_steps(self._handle.count_success(deployment_id)) + except Exception as e: # noqa: BLE001 # as the Rust call loop treats a success it could not record + verbose_router_logger.debug("count_realtime_session_success: not recorded. Got - %s", e) + + +__all__ = [ + "RustHealthCallbacks", + "failure_status", + "logged_call", +] diff --git a/litellm/router_rust/surface/health_routing.py b/litellm/router_rust/surface/health_routing.py new file mode 100644 index 00000000000..46542bf9af2 --- /dev/null +++ b/litellm/router_rust/surface/health_routing.py @@ -0,0 +1,160 @@ +""" +Health-check routing, allowed-fails policies and the failover flags on the host side of the Rust +core: how the constructor resolves them, the static document router-core reads them from, and the +``health()`` members that read the registry's live model list or the background health checks' +results. router-core makes every routing decision (the health-check filter, the cooldown rules, +weighted failover); this module only reads Python values into its documents and back. + +The rules are the Python router core's, ported (never imported). +""" + +import dataclasses +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.types.router import AllowedFailsPolicy + +if TYPE_CHECKING: + from litellm.caching.caching import DualCache + from litellm.router_config import RouterConfig + from litellm.types.router import Deployment, DeploymentTypedDict + +HEALTH_STATE_KEY: Final = DeploymentHealthCache.CACHE_KEY +# The failover flags the Python core keeps as plain router attributes, with their defaults. +FAILOVER_FLAGS: Final = ("enable_weighted_failover", "fallback_on_unsupported_params") + + +def resolved_allowed_fails_policy(value: object) -> AllowedFailsPolicy | None: + """The router's ``allowed_fails_policy`` as the constructor keeps it: a dict is read into an + ``AllowedFailsPolicy`` (a bad field raises the model's own error), a policy is kept, anything + else is no policy.""" + if isinstance(value, AllowedFailsPolicy): + return value + if isinstance(value, dict): + return AllowedFailsPolicy.model_validate(value) + return None + + +def _staleness(threshold: object) -> float: + """``health_check_staleness_threshold or DEFAULT_HEALTH_CHECK_INTERVAL * multiplier``.""" + from litellm.constants import DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER + + given: Final = threshold if isinstance(threshold, int | float) and not isinstance(threshold, bool) else None + return float(given or (DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER)) + + +@dataclasses.dataclass(frozen=True, slots=True) +class HealthRouting: + """The health-check routing settings, as the constructor resolves them.""" + + enable_health_check_routing: bool = False + health_check_staleness: float = 600.0 + health_check_ignore_transient_errors: bool = False + background_health_check_model_groups: frozenset[str] | None = None + + @classmethod + def of(cls, config: "RouterConfig") -> "HealthRouting": + groups: Final = config.get("background_health_check_model_groups") + return cls( + enable_health_check_routing=bool(config.get("enable_health_check_routing")), + health_check_staleness=_staleness(config.get("health_check_staleness_threshold")), + health_check_ignore_transient_errors=bool(config.get("health_check_ignore_transient_errors")), + background_health_check_model_groups=frozenset(groups) if groups is not None else None, # pyright: ignore[reportArgumentType, reportUnknownArgumentType] # any iterable of names, as the Python core takes it + ) + + def static_document(self) -> Mapping[str, object]: + groups: Final = self.background_health_check_model_groups + return { + "enable_health_check_routing": True if self.enable_health_check_routing else None, + "health_check_staleness_threshold": self.health_check_staleness, + # Only a string can name a model group, so anything else in scope matches nothing. + "background_health_check_model_groups": sorted(name for name in groups if isinstance(name, str)) # pyright: ignore[reportUnnecessaryIsInstance] # `of` keeps any iterable the constructor was given + if groups is not None + else None, + } + + def health_checks(self, cache: "DualCache") -> DeploymentHealthCache: + return DeploymentHealthCache(cache=cache, staleness_threshold=self.health_check_staleness) + + +def failover_document(plain: Mapping[str, object]) -> Mapping[str, object]: + """The failover flags router-core reads, from the router's plain attributes (read for + truthiness, as the Python core's checks read them).""" + return { + "enable_weighted_failover": True if plain.get("enable_weighted_failover") else None, + "fallback_on_unsupported_params": False if not plain.get("fallback_on_unsupported_params", True) else None, + } + + +def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]: + return as_str_mapping(deployment.get("model_info")) or {} + + +def is_blocked(deployment: "Deployment") -> bool: + """``_core_is_deployment_blocked``.""" + model_info: Final[object | None] = getattr(deployment, "model_info", None) + if model_info is None: + return False + return getattr(model_info, "blocked", None) is True + + +def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: + """``_core_are_all_deployments_blocked``.""" + return len(deployments) > 0 and all( + _model_info(deployment).get("blocked") is True # pyright: ignore[reportArgumentType] # a deployment dict + for deployment in deployments + ) + + +def fully_blocked_model_names( + model_list: Sequence[Mapping[str, object]], +) -> set[str]: # mutable-ok: the core's type, a fresh set + """``_core_get_fully_blocked_model_names``: groups whose every deployment is blocked.""" + blocked_by_name: Final[dict[str, bool]] = {} # mutable-ok: built here + for deployment in model_list: + name = deployment.get("model_name") or "" + if not isinstance(name, str) or not name: + continue + blocked = _model_info(deployment).get("blocked") is True + blocked_by_name[name] = blocked_by_name.get(name, True) and blocked + return {name for name, fully in blocked_by_name.items() if fully} + + +async def fully_unhealthy_model_names( + model_list: Sequence[Mapping[str, object]], health_checks: DeploymentHealthCache, policy: object +) -> set[str]: # mutable-ok: the core's type, a fresh set + """``_core_async_get_fully_unhealthy_model_names``: groups (and team public names) whose + every deployment failed its health check; nothing under an allowed_fails_policy.""" + if policy is not None: + return set() + unhealthy_ids: Final = await health_checks.async_get_unhealthy_deployment_ids() + if not unhealthy_ids: + return set() + unhealthy_by_name: Final[dict[str, bool]] = {} # mutable-ok: built here + for deployment in model_list: + model_info = _model_info(deployment) + names = [deployment.get("model_name") or ""] + team_public_model_name = model_info.get("team_public_model_name") + if team_public_model_name: + names.append(team_public_model_name) + unhealthy = model_info.get("id") in unhealthy_ids + for name in names: + if not isinstance(name, str) or not name: + continue + unhealthy_by_name[name] = unhealthy_by_name.get(name, True) and unhealthy + return {name for name, fully in unhealthy_by_name.items() if fully} + + +__all__ = [ + "FAILOVER_FLAGS", + "HEALTH_STATE_KEY", + "HealthRouting", + "all_blocked", + "failover_document", + "fully_blocked_model_names", + "fully_unhealthy_model_names", + "is_blocked", + "resolved_allowed_fails_policy", +] diff --git a/litellm/router_rust/surface/indexes.py b/litellm/router_rust/surface/indexes.py new file mode 100644 index 00000000000..f5aca3e1bad --- /dev/null +++ b/litellm/router_rust/surface/indexes.py @@ -0,0 +1,514 @@ +# pyright: reportIncompatibleMethodOverride=false +# (each view overrides its builtin's mutators with the same behaviour, as a changeset) +""" +``HostedIndexes``: the registry indexes the Rust core hands out, read back from router-core. + +router-core keeps the indexes the Python router keeps by history (``host_index.rs``): the id, +group and team position maps, the group names a ``set_model_list`` was given, the deployment +names that only ever grow, the team public names, the provider default ids and the wildcard +pattern routers. ``registry().index()``, ``names(kind=...)``, ``patterns()`` and the deprecated +attributes return the views here: a ``dict``, ``set`` or ``list`` (so ``isinstance`` and ``json`` +keep working), one object per router, whose contents are router-core's, read again when an entry +point hands a view out (or before a write to it) once the registry moved, not on every registry +write. A write to any of them, a nested one included (``index("model_name")["gpt"].append(3)``), +is a ``host_index`` changeset that replaces that index in router-core, and the view reads it back. +The lookups over these indexes run in router-core (``lookup.rs``), not over the views. +""" + +import copy +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from functools import partial +from typing import ( + TYPE_CHECKING, + Final, + Literal, + SupportsIndex, + TypeAlias, + cast, # noqa: TID251 # any value a caller wrote into a view +) + +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter + +if TYPE_CHECKING: + from litellm.rust_bridge._native import RouterCoreHandle + +IndexField: TypeAlias = Literal[ + "id_map", + "model_name", + "team_model", + "model_names", + "deployment_names", + "team_public_model_names", + "provider_default_ids", + "patterns", + "team_patterns", +] +_Commit: TypeAlias = Callable[[object], None] +# Deployment copies a pattern router routes to, fresh for the caller. +Routed: TypeAlias = list[dict[str, object]] +_Pairs: TypeAlias = list[tuple[str, list[object]]] + + +def _stay() -> None: + """A view with no contents of its own to read again before a write.""" + + +def _as_set(value: object) -> set[object]: # mutable-ok: set's own in-place operand + return cast(set[object], value) # cast-ok: an in-place set operand, checked by set itself + + +def _as_mapping(value: object) -> Mapping[object, object]: + if not isinstance(value, Mapping): + return {} + return cast(Mapping[object, object], value) # cast-ok: any mapping a caller wrote + + +def _json(value: object, depth: int = 0) -> object: + """A view's contents as router-core takes them: plain JSON, a set as a list. A caller wrote them, + so a cycle is possible: anything nested past ``DEFAULT_MAX_RECURSE_DEPTH`` is refused.""" + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError(f"a router index is nested deeper than {DEFAULT_MAX_RECURSE_DEPTH} levels") + mapping: Final = _as_mapping(value) + if mapping or isinstance(value, Mapping): + return {str(key): _json(item, depth + 1) for key, item in mapping.items()} + if isinstance(value, list | tuple | set | frozenset): + items: Final = cast(Iterable[object], value) # cast-ok: any collection a caller wrote + return [_json(item, depth + 1) for item in items] + return value + + +class _Positions(list[object]): + """One index's position list (or a pattern's deployments): a write commits the whole list.""" + + def __init__(self, items: Iterable[object], commit: _Commit, fresh: Callable[[], None] = _stay) -> None: + super().__init__(items) + self._commit: Final = commit + self._fresh: Final = fresh + + def _load(self, items: Iterable[object]) -> None: + """Show router-core's contents, without a write.""" + super().__setitem__(slice(None), list(items)) + + def _write(self, change: Callable[[list[object]], object]) -> object: # mutable-ok: the copy it changes + self._fresh() # a view held across a write elsewhere changes router-core's contents, not stale ones + changed: Final = list(self) + result: Final = change(changed) + self._commit(changed) + return result + + def append(self, value: object) -> None: + _ = self._write(lambda items: items.append(value)) + + def extend(self, values: Iterable[object]) -> None: + added: Final = tuple(values) + _ = self._write(lambda items: items.extend(added)) + + def insert(self, index: SupportsIndex, value: object) -> None: + _ = self._write(lambda items: items.insert(index, value)) + + def remove(self, value: object) -> None: + _ = self._write(lambda items: items.remove(value)) + + def pop(self, index: SupportsIndex = -1) -> object: + return self._write(lambda items: items.pop(index)) + + def clear(self) -> None: + _ = self._write(lambda items: items.clear()) + + def sort(self, *args: object, **kwargs: object) -> None: # kwargs-ok: list.sort's own arguments + _ = self._write(lambda items: items.sort(*args, **kwargs)) # pyright: ignore[reportCallIssue, reportArgumentType] # list.sort's own arguments + + def reverse(self) -> None: + _ = self._write(lambda items: items.reverse()) # pyright: ignore[reportUnknownLambdaType] # list.reverse returns None + + def __setitem__(self, index: object, value: object) -> None: + _ = self._write(lambda items: items.__setitem__(index, value)) # pyright: ignore[reportCallIssue, reportArgumentType, reportUnknownLambdaType] # list's own item and slice assignment + + def __delitem__(self, index: object) -> None: + _ = self._write(lambda items: items.__delitem__(index)) # pyright: ignore[reportArgumentType] # list's own item and slice deletion + + def __iadd__(self, values: Iterable[object]) -> "_Positions": + self.extend(values) + return self + + def __imul__(self, count: SupportsIndex) -> "_Positions": + _ = self._write(lambda items: items.__imul__(count)) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (list, (list(self),)) + + +class _IndexDict(dict[object, object]): + """A key-to-positions index: a write commits the whole index.""" + + def __init__(self, commit: _Commit, fresh: Callable[[], None] = _stay) -> None: + super().__init__() + self._commit: Final = commit + self._fresh: Final = fresh + + def _load(self, items: Mapping[object, object]) -> None: + """Show router-core's contents, without a write.""" + super().clear() + super().update(items) + + def _write(self, change: Callable[[dict[object, object]], object]) -> object: # mutable-ok: the copy it changes + self._fresh() # a view held across a write elsewhere changes router-core's contents, not stale ones + changed: Final = dict(self) + result: Final = change(changed) + self._commit(changed) + return result + + def __setitem__(self, key: object, value: object) -> None: + _ = self._write(lambda items: items.__setitem__(key, value)) + + def __delitem__(self, key: object) -> None: + _ = self._write(lambda items: items.__delitem__(key)) + + def pop(self, key: object, *default: object) -> object: + return self._write(lambda items: items.pop(key, *default)) + + def popitem(self) -> tuple[object, object]: + return cast(tuple[object, object], self._write(lambda items: items.popitem())) # cast-ok: dict.popitem's pair + + def clear(self) -> None: + _ = self._write(lambda items: items.clear()) + + def update(self, *args: object, **kwargs: object) -> None: # kwargs-ok: dict.update's own arguments + _ = self._write(lambda items: items.update(*args, **kwargs)) # pyright: ignore[reportCallIssue, reportArgumentType] # dict.update's own arguments + + def setdefault(self, key: object, default: object = None) -> object: + if key in self: + return self[key] + self[key] = default + return self[key] + + def __ior__(self, other: object) -> "_IndexDict": + self.update(other) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (dict, (dict(self),)) + + +class _NameSet(set[object]): + """``model_names``: a write commits the whole set.""" + + def __init__(self, commit: _Commit, fresh: Callable[[], None] = _stay) -> None: + super().__init__() + self._commit: Final = commit + self._fresh: Final = fresh + + def _load(self, items: Iterable[object]) -> None: + """Show router-core's contents, without a write.""" + super().clear() + super().update(items) + + def _write(self, change: Callable[[set[object]], object]) -> object: # mutable-ok: the copy it changes + self._fresh() # a view held across a write elsewhere changes router-core's contents, not stale ones + changed: Final = set(self) + result: Final = change(changed) + self._commit(changed) + return result + + def add(self, value: object) -> None: + _ = self._write(lambda items: items.add(value)) + + def discard(self, value: object) -> None: + _ = self._write(lambda items: items.discard(value)) + + def remove(self, value: object) -> None: + _ = self._write(lambda items: items.remove(value)) + + def pop(self) -> object: + return self._write(lambda items: items.pop()) + + def clear(self) -> None: + _ = self._write(lambda items: items.clear()) + + def update(self, *others: Iterable[object]) -> None: + _ = self._write(lambda items: items.update(*others)) + + def difference_update(self, *others: Iterable[object]) -> None: + _ = self._write(lambda items: items.difference_update(*others)) + + def intersection_update(self, *others: Iterable[object]) -> None: + _ = self._write(lambda items: items.intersection_update(*others)) + + def symmetric_difference_update(self, other: Iterable[object]) -> None: + _ = self._write(lambda items: items.symmetric_difference_update(other)) + + def __ior__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__ior__(_as_set(other))) + return self + + def __iand__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__iand__(_as_set(other))) + return self + + def __isub__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__isub__(_as_set(other))) + return self + + def __ixor__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__ixor__(_as_set(other))) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (set, (set(self),)) + + +class HostedPatternRouter(PatternMatchRouter): + """A wildcard pattern router whose patterns are router-core's (``pattern_router``, or one team's + router): routing runs in router-core, and ``add_pattern`` / ``remove_deployment`` / a write to + ``patterns`` are changesets.""" + + def __init__(self, owner: "HostedIndexes", team: str | None) -> None: + super().__init__() # its empty `patterns` write is ignored: router-core holds them + self._owner: Final = owner + self._team: Final = team + self._view: Final = _IndexDict(self._commit, owner.refresh) + + @property + def patterns(self) -> dict[str, list[object]]: # pyright: ignore[reportIncompatibleVariableOverride] # the base class's attribute, read from router-core # mutable-ok: the live patterns, writes are changesets + self._owner.refresh() + return cast(dict[str, list[object]], self._view) # cast-ok: regex keys, deployment lists + + @patterns.setter + def patterns(self, value: Mapping[str, object]) -> None: # pyright: ignore[reportIncompatibleVariableOverride] # see the getter + if not hasattr(self, "_owner"): + return # the base constructor's empty dict: router-core holds this router's patterns + self._commit(value) + + def _show(self, pairs: Iterable[tuple[str, Sequence[object]]]) -> None: + shown: Final = { + regex: _Positions(deployments, partial(self._write_pattern, regex)) for regex, deployments in pairs + } + self._view._load(cast(Mapping[object, object], shown)) # pyright: ignore[reportPrivateUsage] # cast-ok: regex to deployments + + def _write_pattern(self, regex: str, deployments: object) -> None: + self._commit({**self.patterns, regex: deployments}) + + def _commit(self, value: object) -> None: + pairs: Final = [ + [str(regex), _json(deployments)] for regex, deployments in (as_str_mapping(value) or {}).items() + ] + self._owner.write_router(self._team, pairs) + + def route(self, request: str | None, filtered_model_names: Sequence[str] | None = None) -> Routed | None: # pyright: ignore[reportIncompatibleMethodOverride] # the base's list types, read only here + if not isinstance(request, str): + # The base class matches its regexes against it, which fails and routes nowhere. + return super().route(request, None if filtered_model_names is None else list(filtered_model_names)) # pyright: ignore[reportReturnType, reportUnknownMemberType, reportUnknownVariableType] # None, or the base's copies of the deployments + return self._owner.route(self._team, request, filtered_model_names) + + def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model goes to the base class + return super().get_models_by_pattern(model, custom_llm_provider) + provider: Final = custom_llm_provider or inferred_provider(model) + return self._owner.models_by_pattern(self._team, [model, f"{provider}/{model}"]) + + +class _TeamRouters(dict[str, HostedPatternRouter]): + """``team_pattern_routers``: team id to its router; removing a team's router is a changeset.""" + + def __init__(self, owner: "HostedIndexes") -> None: + super().__init__() + self._owner: Final = owner + + def _load(self, routers: Mapping[str, HostedPatternRouter]) -> None: + super().clear() + super().update(routers) + + def __delitem__(self, team: str) -> None: + self._owner.drop_team_router(team) + + def pop(self, team: str, *default: object) -> object: + if team not in self: + if default: + return default[0] + raise KeyError(team) + router: Final = self[team] + self._owner.drop_team_router(team) + return router + + def clear(self) -> None: + self._owner.write({"team_patterns": []}) + + def setdefault(self, team: str, default: HostedPatternRouter | None = None) -> HostedPatternRouter: + if team not in self: + self._owner.write_router(team, []) + return self[team] + + +class HostedIndexes: + """Router-core's host indexes, as the live objects the registry hands out.""" + + def __init__( + self, + handle: Callable[[], "RouterCoreHandle | None"], + apply: Callable[[Mapping[str, object]], None], + ) -> None: + self._handle: Final = handle + self._apply: Final = apply + self._read_at: tuple[object, int] | None = None + self._document: Mapping[str, object] = {} + # The views are read again when an entry point hands them out, and before a write to them. + self.id_map: Final = _IndexDict(lambda value: self.write({"id_map": value}), self.refresh) + self.model_name: Final = _IndexDict(lambda value: self.write({"model_name": value}), self.refresh) + self.team_model: Final = _IndexDict(lambda value: self._write_team_model(value), self.refresh) + self.model_names: Final = _NameSet(lambda value: self.write({"model_names": value}), self.refresh) + self.deployment_names: Final = _Positions( + (), lambda value: self.write({"deployment_names": value}), self.refresh + ) + self.provider_default_ids: Final = _Positions( + (), lambda value: self.write({"provider_default_ids": value}), self.refresh + ) + self.pattern_router: Final = HostedPatternRouter(self, None) + self.team_pattern_routers: Final = _TeamRouters(self) + self._team_routers: dict[str, HostedPatternRouter] = {} # mutable-ok: one router object per team + self._team_public: frozenset[str] = frozenset() + + # -- reading ---------------------------------------------------------------------------------- + + def refresh(self) -> None: + """Read router-core's indexes again when its registry moved since the last read.""" + handle: Final = self._handle() + if handle is None: + return + version: Final = handle.registry_version() + if self._read_at is not None and self._read_at[0] is handle and self._read_at[1] == version: + return + self._read_at = (handle, version) + self._document = as_str_mapping(handle.host_indexes()) or {} + self._show() + + def _field(self, name: str) -> object: + return self._document.get(name) + + def _show(self) -> None: + ids: Final = _as_mapping(self._field("id_map")) + self.id_map._load(ids) # pyright: ignore[reportPrivateUsage] # a view this object owns + named: Final[dict[object, object]] = {} # mutable-ok: built here + for name, positions in (as_str_mapping(self._field("model_name")) or {}).items(): + named[name] = self._positions(positions, partial(self._write_named, name)) + self.model_name._load(named) # pyright: ignore[reportPrivateUsage] # a view this object owns + teams: Final[dict[object, object]] = {} # mutable-ok: built here + for team, public, positions in _triples(self._field("team_model")): + teams[(team, public)] = self._positions(positions, partial(self._write_team_key, (team, public))) + self.team_model._load(teams) # pyright: ignore[reportPrivateUsage] # a view this object owns + self.model_names._load(_items(self._field("model_names"))) # pyright: ignore[reportPrivateUsage] # a view this object owns + self.deployment_names._load(_items(self._field("deployment_names"))) # pyright: ignore[reportPrivateUsage] # a view this object owns + self.provider_default_ids._load(_items(self._field("provider_default_ids"))) # pyright: ignore[reportPrivateUsage] # a view this object owns + self._team_public = frozenset(str(name) for name in _items(self._field("team_public_model_names"))) + self.pattern_router._show(_pattern_pairs(self._field("patterns"))) # pyright: ignore[reportPrivateUsage] # the router this object owns + routers: Final[dict[str, HostedPatternRouter]] = {} # mutable-ok: rebuilt each read + for team, pairs in _team_pairs(self._field("team_patterns")): + router = self._team_routers.get(team) or HostedPatternRouter(self, team) + router._show(pairs) # pyright: ignore[reportPrivateUsage] # the router this object owns + routers[team] = router + self._team_routers = routers + self.team_pattern_routers._load(routers) # pyright: ignore[reportPrivateUsage] # a view this object owns + + def _write_named(self, name: str, items: object) -> None: + self.write({"model_name": {**self.model_name, name: items}}) + + def _write_team_key(self, key: tuple[str, str], items: object) -> None: + self._write_team_model({**self.team_model, key: items}) + + def _positions(self, positions: object, commit: _Commit) -> object: + """A position list as a write-through list; anything else a caller wrote, as written.""" + items: Final = _items(positions) + return _Positions(items, commit) if isinstance(positions, list) else positions + + @property + def team_public_model_names(self) -> frozenset[str]: + self.refresh() + return self._team_public + + # -- writing ---------------------------------------------------------------------------------- + + def write(self, fields: Mapping[str, object]) -> None: + """Replace the named indexes in router-core, then read them back.""" + self._apply({"host_index": {name: _json(value) for name, value in fields.items()}}) + self.refresh() + + def assign(self, field: IndexField, value: object) -> None: + """A deprecated attribute assignment of one index (``router.model_names = {...}``).""" + if field == "team_model": + self._write_team_model(value) + else: + self.write({field: value}) + + def _write_team_model(self, value: object) -> None: + mapping: Final = _as_mapping(value) + triples: Final[list[list[object]]] = [] # mutable-ok: JSON triples + for key, positions in mapping.items(): + parts = cast(tuple[object, ...], key) if isinstance(key, tuple) else () # cast-ok: a key a caller wrote + if len(parts) == 2: + triples.append([str(parts[0]), str(parts[1]), _json(positions)]) + self.write({"team_model": triples}) + + def write_router(self, team: str | None, pairs: list[list[object]]) -> None: # mutable-ok: JSON pairs + """Replace one pattern router's patterns.""" + if team is None: + self.write({"patterns": pairs}) + return + teams: Final = [[t, _json(_pairs_of(router))] for t, router in self._team_routers.items() if t != team] + self.write({"team_patterns": [*teams, [team, pairs]]}) + + def drop_team_router(self, team: str) -> None: + if team not in self._team_routers: + raise KeyError(team) + self.write({"team_patterns": [[t, _json(_pairs_of(r))] for t, r in self._team_routers.items() if t != team]}) + + # -- routing ---------------------------------------------------------------------------------- + + def route(self, team: str | None, request: str, filtered: Sequence[str] | None) -> Routed | None: + handle: Final = self._handle() + routed: Final = ( + None + if handle is None + else handle.pattern_route(team, request, None if filtered is None else list(filtered)) + ) + return None if routed is None else [dict(deployment) for deployment in routed] + + def models_by_pattern(self, team: str | None, requests: list[str]) -> frozenset[str]: # mutable-ok: read only + handle: Final = self._handle() + found: Final = None if handle is None else handle.models_by_pattern(team, requests) + return frozenset(found or ()) + + +def _items(value: object) -> list[object]: # mutable-ok: a fresh list + return list(cast(list[object], value)) if isinstance(value, list) else [] # cast-ok: a JSON list + + +def _triples(value: object) -> Iterator[tuple[str, str, object]]: + for item in _items(value): + parts = _items(item) + if len(parts) == 3: + yield str(parts[0]), str(parts[1]), parts[2] + + +def _pattern_pairs(value: object) -> _Pairs: + pairs: Final[_Pairs] = [] + for item in _items(value): + parts = _items(item) + if len(parts) == 2: + pairs.append((str(parts[0]), _items(parts[1]))) + return pairs + + +def _team_pairs(value: object) -> Iterator[tuple[str, _Pairs]]: + for item in _items(value): + parts = _items(item) + if len(parts) == 2: + yield str(parts[0]), _pattern_pairs(parts[1]) + + +def _pairs_of(router: HostedPatternRouter) -> list[list[object]]: # mutable-ok: JSON pairs + view: Final[Mapping[object, object]] = router._view # pyright: ignore[reportPrivateUsage] # a router this module owns + return [[regex, copy.deepcopy(deployments)] for regex, deployments in view.items()] diff --git a/litellm/router_rust/surface/legacy_members.py b/litellm/router_rust/surface/legacy_members.py new file mode 100644 index 00000000000..6c30557eb4d --- /dev/null +++ b/litellm/router_rust/surface/legacy_members.py @@ -0,0 +1,547 @@ +""" +``RustLegacyMembers``: the old public ``Router`` members outside the entry points, on the Rust +core: the routing-state lifecycle, the routing callbacks' checks, one provider call's accounting +and the pre-routing strategy builders. Each one is the Python router core's, ported (never +imported), over the state the Rust core and its host keep. + +Not served, and refused by ``RustCore.__getattr__``: the Python fallback walk +(``async_function_with_fallbacks``, ``function_with_fallbacks``, ``async_function_with_retries``, +``async_function_with_fallbacks_common_utils``) and ``factory_function``. They run a caller's +``original_function``, which picks its own deployment; router-core makes each pick itself and +attempts one host op, named by the endpoint's plan. +""" + +import asyncio +import contextlib +import inspect +import traceback +import uuid +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.add_retry_fallback_headers import response_total_token_count +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_parent_otel_span_from_kwargs +from litellm.litellm_core_utils.coroutine_checker import coroutine_checker +from litellm.litellm_core_utils.routing_groups import validate_routing_strategy +from litellm.litellm_core_utils.token_counter import served_openai_models as _served_openai_models +from litellm.router_api import RoutingContext +from litellm.router_rust.invoke.websocket import admit_deployment_turn +from litellm.router_rust.steps.callbacks import RustOptionalChecks +from litellm.router_rust.steps.prerouting import TIER_PARAMS_NEVER_DROPPED as _TIER_PARAMS_NEVER_DROPPED +from litellm.router_rust.surface.members import RustRegistryMembers +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + GatedAutoRouterCapability, + capability_limit_violation, + count_capability_routers, +) +from litellm.router_utils.cooldown_handlers import set_cooldown_deployments +from litellm.router_utils.endpoint_eligibility import endpoint_selection +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_io_token_refund_failure +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET +from litellm.types.router import RouterCacheEnum +from litellm.utils import get_utc_datetime + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + from litellm.caching.dual_cache import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.router_rust.invoke.endpoints import EndpointCore + from litellm.types.router import CustomRoutingStrategyBase, Deployment, RoutingGroup + +# `RoutingArgs.ttl`: a usage counter lives for its minute. +_USAGE_TTL: Final = 60 + +# What `initialize_assistants_endpoint` binds. +_ASSISTANTS_ENDPOINTS: Final = ( + "acreate_assistants", + "adelete_assistant", + "aget_assistants", + "acreate_thread", + "aget_thread", + "a_add_message", + "aget_messages", + "arun_thread", +) + +# What `initialize_router_endpoints` binds, with the container file endpoints it reads from +# `litellm/containers/endpoints.json`. +_ROUTER_ENDPOINTS: Final = frozenset( + { + "_arealtime", + "_aresponses_websocket", + "aadapter_generate_content", + "aanthropic_messages", + "acancel_fine_tuning_job", + "acancel_interaction", + "acancel_responses", + "acompact_responses", + "acreate_agent", + "acreate_container", + "acreate_fine_tuning_job", + "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "acreate_skill", + "adecisions", + "adelete_agent", + "adelete_container", + "adelete_interaction", + "adelete_responses", + "adelete_skill", + "aextraction", + "afile_content", + "afile_delete", + "afile_list", + "agenerate_content", + "agenerate_content_stream", + "aget_agent", + "aget_interaction", + "aget_responses", + "aget_skill", + "aimage_edit", + "alist_agent_versions", + "alist_agents", + "alist_containers", + "alist_fine_tuning_jobs", + "alist_input_items", + "alist_skills", + "allm_passthrough_route", + "amoderation", + "anthropic_messages", + "aocr", + "arealtime_calls", + "arealtime_translation_calls", + "aresponses", + "aretrieve_container", + "aretrieve_fine_tuning_job", + "asearch", + "asystemone", + "avector_store_delete", + "avector_store_file_content", + "avector_store_file_create", + "avector_store_file_delete", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_update", + "avector_store_list", + "avector_store_retrieve", + "avector_store_search", + "avector_store_update", + "avideo_content", + "avideo_create_character", + "avideo_edit", + "avideo_extension", + "avideo_generation", + "avideo_get_character", + "avideo_list", + "avideo_remix", + "avideo_status", + "cancel_interaction", + "create_agent", + "create_container", + "create_interaction", + "decisions", + "delete_agent", + "delete_container", + "delete_interaction", + "extraction", + "generate_content", + "generate_content_stream", + "get_agent", + "get_interaction", + "list_agent_versions", + "list_agents", + "list_containers", + "ocr", + "responses", + "retrieve_container", + "search", + "systemone", + "vector_store_create", + "vector_store_delete", + "vector_store_file_content", + "vector_store_file_create", + "vector_store_file_delete", + "vector_store_file_list", + "vector_store_file_retrieve", + "vector_store_file_update", + "vector_store_list", + "vector_store_retrieve", + "vector_store_search", + "vector_store_update", + "video_content", + "video_create_character", + "video_edit", + "video_extension", + "video_generation", + "video_get_character", + "video_list", + "video_remix", + "video_status", + } +) + + +def _dispatch_failure(logging_obj: "Logging | None", error: Exception) -> None: + """The ``## LOG FAILURE EVENT`` of a routing callback that raised.""" + if logging_obj is not None: + asyncio.create_task( + logging_obj.dispatch_failure_handlers( + exception=error, + traceback_exception=traceback.format_exc(), + prefer_async_handlers=True, + ) + ) + + +class _FailedCallParams(TypedDict): + """The ``litellm_params`` of a failed call, as the failure callback reads them.""" + + metadata: ReadOnly[Mapping[str, object]] + model_info: ReadOnly[NotRequired[Mapping[str, object] | None]] + + +async def _deployments_filtered_by( + callback: CustomLogger, + model: str, + deployments: object, + messages: object, + request_kwargs: dict[str, object] | None, # mutable-ok: CustomLogger takes the request dict + parent_otel_span: "Span | None", +) -> object: + return await callback.async_filter_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts + model=model, + healthy_deployments=deployments, # pyright: ignore[reportArgumentType] # what the last callback returned + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + request_kwargs=request_kwargs, + parent_otel_span=parent_otel_span, + ) + + +def _metadata_model_id(model_info: object) -> object: + return model_info.get("id") if isinstance(model_info, dict) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the metadata's model_info dict + + +class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): + """Mixed into ``RustCore``, which supplies its host registry, selectors and checks.""" + + # RustCore sets these in its constructor. + _cache: "DualCache" + _cooldown_given: object + _custom_strategy: "CustomRoutingStrategyBase | None" + # The constructor's argument, or the attribute as assigned. + auto_router_capability_limit: "Callable[[], int | None] | None" + + TIER_PARAMS_NEVER_DROPPED: Final = _TIER_PARAMS_NEVER_DROPPED + + # -- registry lookups -------------------------------------------------------------------------- + + def get_routing_group(self, model_name: str) -> "RoutingGroup | None": + """The routing group callable as ``model_name`` (``get_routing_group``).""" + return self._registry_state.get_routing_group(model_name) + + def team_public_model_names_of(self, team_id: str) -> frozenset[str]: + """The public names ``team_id`` serves its own deployments under (``team_public_model_names_of``).""" + return self._registry_state.team_public_model_names_of(team_id) + + def served_openai_models(self, model_name: str) -> tuple[str | None, ...]: + """The OpenAI models ``model_name`` serves (``_served_openai_models``). The Python core keeps + an ``lru_cache`` over it, emptied with every model list change.""" + return _served_openai_models(self._registry_state._core_deployments_named(model_name) or ()) # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + + def auto_router_capability_violation(self, capability: GatedAutoRouterCapability) -> str | None: + """Why one more router claiming ``capability`` cannot join this router, or None when it can + (``auto_router_capability_violation``); the limit is resolved on every call.""" + resolver: Final = self.auto_router_capability_limit + limit: Final = resolver() if resolver is not None else None + others: Final = count_capability_routers( + (deployment for deployment in self._registry_state._core_model_list if isinstance(deployment, Mapping)), # pyright: ignore[reportPrivateUsage, reportUnnecessaryIsInstance] # the live list + capability=capability, + ) + return capability_limit_violation(capability=capability, held=others + 1, limit=limit) + + # -- routing state ----------------------------------------------------------------------------- + + def retire_routing_state(self) -> None: + """Stop the background work of the selectors and the budget limiter (``retire_routing_state``).""" + self._opt.retire_state((*self._sel.callback_selectors(), *self._sel.override_selectors())) + + def routing_strategy_init(self, routing_strategy: object, routing_strategy_args: Mapping[str, object]) -> None: + """``routing_strategy_init``: the default selector only. As on the Python core the router's + ``routing_strategy`` setting stays, so picks keep following it; ``apply`` changes it.""" + verbose_router_logger.info("Routing strategy: %s", routing_strategy) + validate_routing_strategy(routing_strategy) # pyright: ignore[reportArgumentType] # the strategy as given + # `_reset_custom_routing_strategy`. + self._custom_strategy = None + _ = self.__dict__.pop("get_available_deployment", None) + _ = self.__dict__.pop("_core_async_get_available_deployment", None) + self._sel.init_default(routing_strategy, routing_strategy_args, configured=False) + + def set_optional_pre_call_checks(self, optional_pre_call_checks: Iterable[object] | None) -> None: + """``set_optional_pre_call_checks``: drop the toggleable checks the list leaves out, then add it.""" + self._opt.set(None if optional_pre_call_checks is None else tuple(optional_pre_call_checks)) + + def initialize_assistants_endpoint(self) -> None: + """``initialize_assistants_endpoint`` binds the endpoints; this core answers them from + ``__getattr__``, so what is left to do is to drop an assigned replacement.""" + for name in _ASSISTANTS_ENDPOINTS: + _ = self.__dict__.pop(name, None) + + def initialize_router_endpoints(self) -> None: + """``initialize_router_endpoints``, as ``initialize_assistants_endpoint`` above.""" + from litellm.containers.endpoint_factory import ( + _generated_endpoints, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # the names the Python core registers + ) + + for name in (*_ROUTER_ENDPOINTS, *_generated_endpoints): + _ = self.__dict__.pop(name, None) + + # -- pre-routing strategies -------------------------------------------------------------------- + + def init_auto_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's auto-router (``init_auto_router_deployment``).""" + self._registry_state.strategies._init_auto_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # RustCore installs its strategies in its constructor + + def init_complexity_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's complexity-router (``init_complexity_router_deployment``).""" + self._registry_state.strategies._init_complexity_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # see above + + def init_quality_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's quality-router (``init_quality_router_deployment``).""" + self._registry_state.strategies._init_quality_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # see above + + def init_adaptive_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's adaptive-router and its post-call hook (``init_adaptive_router_deployment``).""" + self._registry_state.strategies._init_adaptive_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # see above + + # -- routing callbacks ------------------------------------------------------------------------- + + def routing_strategy_pre_call_checks( + self, + deployment: dict[str, object], # mutable-ok: CustomLogger takes the deployment dict + ) -> None: + """``routing_strategy_pre_call_checks``: the sync check of each ``CustomLogger`` callback.""" + for callback in self._opt.routing_callbacks(): + if isinstance(callback, CustomLogger): + callback.pre_call_check(deployment) # pyright: ignore[reportUnknownMemberType] # CustomLogger takes an untyped deployment dict + + async def async_routing_strategy_pre_call_checks( + self, + deployment: dict[str, object], # mutable-ok: CustomLogger takes the deployment dict + parent_otel_span: "Span | None", + logging_obj: "Logging | None" = None, + ) -> None: + """``async_routing_strategy_pre_call_checks``: a check's rate limit cools the deployment down + under the Python rules (``set_cooldown_deployments``, cooldown event included).""" + from litellm import exceptions as litellm_exceptions + + for callback in self._opt.routing_callbacks(): + if isinstance(callback, CustomLogger): + try: + await callback.async_pre_call_check(deployment, parent_otel_span) # pyright: ignore[reportUnknownMemberType] # see above + except litellm_exceptions.RateLimitError as e: + _dispatch_failure(logging_obj, e) + _ = set_cooldown_deployments( + ctx=RoutingContext(self), # pyright: ignore[reportArgumentType] # the core's RoutingContext + exception_status=e.status_code, + original_exception=e, + deployment=deployment["model_info"]["id"], # pyright: ignore[reportIndexIssue, reportUnknownArgumentType] # read as the Python core reads it + time_to_cooldown=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + ) + raise + except Exception as e: + _dispatch_failure(logging_obj, e) + raise + + async def async_callback_filter_deployments( + self, + model: str, + healthy_deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: object, + parent_otel_span: "Span | None", + request_kwargs: dict[str, object] | None = None, # mutable-ok: CustomLogger takes the request dict + logging_obj: "Logging | None" = None, + ) -> object: + """``async_callback_filter_deployments``: each routing callback narrows the deployments in turn.""" + remaining: object = healthy_deployments # rebind-ok: narrowed by each callback + for callback in self._opt.routing_callbacks(): + if isinstance(callback, CustomLogger): + try: + remaining = await _deployments_filtered_by( + callback, model, remaining, messages, request_kwargs, parent_otel_span + ) + except Exception as e: + _dispatch_failure(logging_obj, e) + raise + return remaining + + async def async_admit_deployment_turn( + self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + """Admit a later turn of an open websocket (``async_admit_deployment_turn``).""" + port: Final[EndpointCore] = self.__dict__["_port"] # pyright: ignore[reportAny] # RustCore's own attribute + return await admit_deployment_turn(port, model_group, opened_on, request_kwargs) + + # -- one provider call ------------------------------------------------------------------------- + + async def make_call( + self, + original_function: Callable[..., object], + *args: object, + **kwargs: object, # kwargs-ok: the endpoint's request + ) -> object: + """``make_call``: call the endpoint, count its usage and stamp the response's headers. A + failed call hands back the io tokens it reserved.""" + model_group: Final = kwargs.get("model") + generic_function: Final = kwargs.get("original_generic_function") + endpoint_function: Final = generic_function if callable(generic_function) else original_function + try: + with endpoint_selection( + getattr(endpoint_function, "__name__", None), + getattr(endpoint_function, "__module__", None), + model=model_group, + ): + response: object = original_function(*args, **kwargs) # rebind-ok: awaited below + if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): + awaitable: Final[Awaitable[object]] = response # pyright: ignore[reportAssignmentType] # the endpoint's awaitable + response = await awaitable # rebind-ok: awaited here + except Exception: + with contextlib.suppress(Exception): + await async_io_token_refund_failure( + self._cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) + ) + raise + await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) + return await self.set_response_headers( + response=response, model_group=model_group if isinstance(model_group, str) else None, request_kwargs=kwargs + ) + + async def increment_deployment_usage_for_response( + self, response: object, request_kwargs: Mapping[str, object] + ) -> None: + """Count a response's tokens and request against its deployment, once the call returned, and + note them on the request for the success callback (``increment_deployment_usage_for_response``).""" + if response is None: + return + try: + deployment_metadata: Final = find_deployment_metadata(request_kwargs) + model_group: Final = request_kwargs.get("model") + if deployment_metadata is None or not isinstance(model_group, str): + return + deployment_id: Final = _metadata_model_id(deployment_metadata["model_info"]) + if deployment_id is None: + return + total_tokens: Final = response_total_token_count(response) + deployment_name: Final = deployment_metadata.get("deployment") + deployment_metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] = total_tokens + try: + await self._increment_deployment_usage( + deployment_id=str(deployment_id), + deployment_name=deployment_name if isinstance(deployment_name, str) else None, + model_group=model_group, + total_tokens=total_tokens, + parent_otel_span=get_parent_otel_span_from_kwargs(dict(request_kwargs)), + ) + except Exception: + _ = deployment_metadata.pop(ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, None) + raise + except Exception as e: # noqa: BLE001 # as the Python core: usage accounting never fails a served call + verbose_router_logger.debug( + "litellm.router.Router::increment_deployment_usage_for_response(): Exception occured - %s", e + ) + + async def _increment_deployment_usage( + self, + *, + deployment_id: str, + deployment_name: str | None, + model_group: str, + total_tokens: float, + parent_otel_span: "Span | None", + ) -> None: + """``_increment_deployment_usage`` with its one request, in one pipeline; this worker's memory + then holds what Redis answered.""" + from litellm.types.caching import RedisPipelineIncrementOperation + + if not self._registry_state.cached_deployment_tracks_usage(deployment_id, model_group): + return + current_minute: Final = get_utc_datetime().strftime("%H-%M") + tpm_key: Final = RouterCacheEnum.TPM.value.format( + id=deployment_id, current_minute=current_minute, model=deployment_name + ) + rpm_key: Final = RouterCacheEnum.RPM.value.format( + id=deployment_id, current_minute=current_minute, model=deployment_name + ) + operations: Final[Sequence[RedisPipelineIncrementOperation]] = [ + RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=_USAGE_TTL) + for key, increment_value in ((tpm_key, total_tokens), (rpm_key, 1)) + ] + values: Final = await self._cache.async_increment_cache_pipeline( + increment_list=list(operations), parent_otel_span=parent_otel_span + ) + if values is not None and self._cache.redis_cache is not None: + for operation, value in zip(operations, values): + await self._cache.async_set_cache(operation["key"], int(value), local_only=True, ttl=_USAGE_TTL) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + + async def async_deployment_callback_on_failure( + self, kwargs: Mapping[str, object], completion_response: object, start_time: object, end_time: object + ) -> None: + """Count a failed call as one more request of the minute (``async_deployment_callback_on_failure``).""" + if is_batch_retrieve_call_type(kwargs.get("call_type")): + return + litellm_params: Final[_FailedCallParams] = kwargs["litellm_params"] # pyright: ignore[reportAssignmentType] # read as the Python core reads it, a missing key included + deployment_name: Final = litellm_params["metadata"].get("deployment", None) + model_group: Final = litellm_params["metadata"].get("model_group", None) + model_info: Final = litellm_params.get("model_info", {}) or {} + deployment_id: Final = model_info.get("id", None) + if model_group is None or deployment_id is None: + return + rpm_key: Final = RouterCacheEnum.RPM.value.format( + id=deployment_id, + current_minute=get_utc_datetime().strftime("%H-%M"), + model=deployment_name, + ) + with service_target(ROUTER_USAGE_TARGET): + await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + key=rpm_key, + value=1, + parent_otel_span=get_parent_otel_span_from_kwargs(dict(kwargs)), + ttl=_USAGE_TTL, + ) + + # -- the plugin context ------------------------------------------------------------------------ + + def _update_kwargs_before_fallbacks( + self, + model: str, + kwargs: dict[str, object], # mutable-ok: the core updates the request kwargs in place + metadata_variable_name: str | None = "metadata", + ) -> None: + """``_update_kwargs_before_fallbacks``: a trace id and the requested group, in place.""" + _ = kwargs.setdefault("litellm_trace_id", str(uuid.uuid4())) + alias: Final = model if self._registry_state._core_get_model_from_alias(model=model) else None # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + kwargs.setdefault(metadata_variable_name, {}).update({"model_group": model, "model_group_alias": alias}) # pyright: ignore[reportArgumentType, reportAttributeAccessIssue, reportUnknownMemberType] # the bucket, as the Python core writes it + + def _params_sent_to_deployment( + self, request_kwargs: Mapping[str, object], deployment: "Deployment | None" + ) -> Mapping[str, object]: + """``_params_sent_to_deployment`` of a ``Deployment``, which ``RoutingContext.deployment_params`` hands in.""" + params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} + configured: Final = as_str_mapping(RoutingContext(self).settings().default_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownArgumentType] # the core's RoutingContext + defaults: Final = {k: v for k, v in (configured or {}).items() if v is not None} + tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {} + return MappingProxyType({**params, **defaults, **tool_choice, **request_kwargs}) + + +__all__ = ["RustLegacyMembers"] diff --git a/litellm/router_rust/surface/members.py b/litellm/router_rust/surface/members.py new file mode 100644 index 00000000000..0f7dc23d966 --- /dev/null +++ b/litellm/router_rust/surface/members.py @@ -0,0 +1,163 @@ +""" +``RustRegistryMembers``: the Rust core's public registry members outside ``registry()`` and +``apply``: ``add_deployment``, ``set_model_list``, the ``model_list`` attribute, the environment +gate, the deprecated id hash, ``print_deployment`` and the pre-call checks the plugin context reads. + +Each one works on the core's host registry (``litellm/router_rust/surface/registry.py``), which hands every +change to the Rust core. The rules are the Python router core's, ported (never imported). +""" + +import copy +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker +from litellm.router_api import deployment_id +from litellm.router_rust.steps.precall import count_tokens, pre_call_checks +from litellm.router_rust.surface.registry import HostRegistry, model_info_is_active_for_environment +from litellm.types.router import Deployment + +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + from litellm.router_rust.core import RustCore + + +class _CallCounter: + """``total_calls``, ``success_calls`` or ``fail_calls``: plain attributes on the Python core, + which a write replaces and a ``del`` removes; every call then reads what is there.""" + + def __init__(self, name: str) -> None: + self._name: Final = name + + def __get__(self, core: "RustCore | None", owner: type | None = None) -> object: + if core is None: + return self + try: + return getattr(core._endpoints.counters, self._name) # pyright: ignore[reportPrivateUsage] # the core's own counters + except AttributeError: + raise core._no_attribute(self._name) from None # pyright: ignore[reportPrivateUsage] # see above + + def __set__(self, core: "RustCore", value: object) -> None: + setattr(core._endpoints.counters, self._name, value) # pyright: ignore[reportPrivateUsage] # see __get__ + + def __delete__(self, core: "RustCore") -> None: + try: + delattr(core._endpoints.counters, self._name) # pyright: ignore[reportPrivateUsage] # see __get__ + except AttributeError: + raise core._no_attribute(self._name) from None # pyright: ignore[reportPrivateUsage] # see __get__ + + +class RustRegistryMembers: + """Mixed into ``RustCore``, which supplies its host registry and host.""" + + total_calls: Final = _CallCounter("total_calls") + success_calls: Final = _CallCounter("success_calls") + fail_calls: Final = _CallCounter("fail_calls") + + @property + def _registry_state(self) -> HostRegistry: + raise NotImplementedError + + @property + def _python_host(self) -> "PythonHost": + raise NotImplementedError + + def add_deployment(self, deployment: Deployment) -> Deployment | None: + """Add a deployment unless its id exists (``add_deployment``).""" + return self._registry_state.add_deployment(deployment) + + def set_model_list(self, model_list: Sequence[object]) -> None: + """Replace every deployment (``set_model_list``); aliases and wildcard routes stay.""" + self._registry_state.set_model_list(model_list) + + @property + def model_list(self) -> Sequence[object]: + """The live model list (the deprecated ``model_list`` attribute).""" + return self._registry_state._core_model_list # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + + def _assign_model_list(self, model_list: Sequence[object]) -> None: + """``router.model_list = [...]``: the list is swapped as given, without reindexing, and the + registry version moves once.""" + self._registry_state.replace_model_list(model_list) # pyright: ignore[reportArgumentType] # the list as assigned, as the core keeps it + + @staticmethod + def deployment_is_active_for_environment(deployment: Deployment) -> bool: + """Whether LITELLM_ENVIRONMENT admits the deployment (``deployment_is_active_for_environment``).""" + return model_info_is_active_for_environment(model_info=deployment.model_info) # pyright: ignore[reportArgumentType] # ModelInfo reads like a mapping + + @staticmethod + def generate_model_id(model_group: str, litellm_params: Mapping[str, object]) -> str: + """The deprecated id hash (``generate_model_id``).""" + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the router's deprecation notice + ) + + _note("generate_model_id", "litellm.router_api.deployment_id(model_group, litellm_params)") + return deployment_id(model_group, litellm_params) + + async def set_response_headers( + self, response: object, model_group: str | None = None, request_kwargs: Mapping[str, object] | None = None + ) -> object: + """Stamp the group's remaining usage and the model id on a response (``set_response_headers``).""" + headed: Final[object] = await self._registry_state.set_response_headers( # pyright: ignore[reportAny] # the response as the core hands it back + response, model_group, None if request_kwargs is None else dict(request_kwargs) + ) + return headed + + def replay_model_cost_registrations(self) -> None: + """Re-register the served deployments' cost-map entries (``replay_model_cost_registrations``).""" + self._registry_state.replay_model_cost_registrations() + + def _count_pre_call_check_tokens( + self, + messages: Sequence[Mapping[str, object]] | None, + input: str | list[object] | None, # mutable-ok: the request field, as the Python core takes it + request_kwargs: Mapping[str, object] | None = None, + tokenizer_model: str = "", + deployments: Sequence[Mapping[str, object]] = (), + ) -> int: + """``_count_pre_call_check_tokens``, which context compaction counts with.""" + return count_tokens(self._python_host, messages, input, request_kwargs, tokenizer_model, deployments) + + def _pre_call_checks( + self, + model: str, + healthy_deployments: Sequence[Mapping[str, object]], + messages: object = None, + input: object = None, + request_kwargs: Mapping[str, object] | None = None, + input_token_count: int | None = None, + skip_inline_token_count: bool = False, + request_deployments: Sequence[Mapping[str, object]] | None = None, + ) -> Sequence[Mapping[str, object]]: + """``_pre_call_checks``, which context compaction admits a client-managed request with.""" + return pre_call_checks( + self._python_host, + self._registry_state, # pyright: ignore[reportArgumentType] # the host registry serves the reads + model, + healthy_deployments, + messages, + input, + request_kwargs, + input_token_count, + skip_inline_token_count, + request_deployments, + ) + + def print_deployment(self, deployment: Mapping[str, object]) -> Mapping[str, object]: + """A copy of the deployment, of the type it was given, with its api key masked + (``print_deployment``). A ``Deployment`` and its params read and write like dicts.""" + try: + printed: Final = copy.deepcopy(deployment) + litellm_params: Final = printed["litellm_params"] + if self._python_host.dynamic().redact_user_api_key_info: + masker: Final = SensitiveDataMasker(visible_prefix=2, visible_suffix=0) + printed["litellm_params"] = masker.mask_dict(litellm_params) # pyright: ignore[reportArgumentType, reportIndexIssue] # the deployment's params, written as the Python core writes them + elif "api_key" in litellm_params: # pyright: ignore[reportOperatorIssue] # see the docstring + litellm_params["api_key"] = litellm_params["api_key"][:2] + "*" * 10 # pyright: ignore[reportIndexIssue] # see the docstring + except Exception as e: + verbose_router_logger.debug("Error occurred while printing deployment - %s", e) + raise + else: + return printed diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py new file mode 100644 index 00000000000..f742225e2f4 --- /dev/null +++ b/litellm/router_rust/surface/model_info.py @@ -0,0 +1,871 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnknownParameterType=false, reportMissingParameterType=false, reportAny=false, reportExplicitAny=false, reportMissingTypeArgument=false, reportAttributeAccessIssue=false, reportTypedDictNotRequiredAccess=false +# (ported from the Python router core as it is written there, over untyped litellm helpers) +""" +``HostModelInfo``: model info, group info, usage reads and response headers for the Rust core's +host registry. + +router-core holds the rules: a deployment's model info (``get_router_model_info``, +``get_deployment_model_info``: ``router-core/src/model_info.rs``), a model group's info and the +configured-value reads (``_set_model_group_info``, ``get_model_listing_info``, the configured token +limits, mode, display name and service tiers: ``router-core/src/group_info.rs``), and the usage +counters' keys (``RouterCoreHandle.usage_key``). What lives in this process is answered here as a +query asks for it, at query time: the cost map (``get_model_info``, ``litellm.model_cost``), +provider resolution, the wildcard pattern routes, what the background refresh discovered, and a +group deployment's validated params, supported params, endpoints, fast mode and reasoning efforts. +``ModelGroupInfo`` is built here from the group's first deployment (its validation is the model's) +and takes what router-core's merge assigned. + +Still the Python router core's rules here: the model group alias a group-info query resolves +first (the alias map is the registry's live Python objects), the cache of discovered limits, the +configured ``max_input_tokens`` read (a value echoed from the loaded cost map is skipped), and the +response headers. +""" + +import builtins +import dataclasses +import math +from collections.abc import Callable, Mapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Union, cast # noqa: TID251 # the core's own signatures, ported unchanged + +from pydantic import BaseModel, TypeAdapter +from typing_extensions import overload + +import litellm +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + apply_quality_router_decision_headers, + apply_remaining_usage_headers, + apply_response_model_id, + complexity_router_decision_headers, + ensure_response_additional_headers, + prepare_response_for_header_attachment, +) +from litellm.litellm_core_utils.batch_utils import deployment_supported_endpoints +from litellm.litellm_core_utils.core_helpers import as_str_mapping, coerce_token_limit +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.reasoning_effort_capability import ( + deployment_is_catalog_mapped, + intersect_supported_reasoning_efforts, + resolve_supported_reasoning_efforts, +) +from litellm.litellm_core_utils.routing_common_utils import team_may_use_deployment +from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.bedrock.common_utils import get_bedrock_base_model +from litellm.router_python_host import python_host +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import build_io_token_rate_limit_headers +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET +from litellm.types.router import ( + Deployment, + DeploymentModelListingInfo, + DeploymentTypedDict, + DiscoveredDeploymentModelInfo, + LiteLLM_Params, + ModelGroupInfo, +) +from litellm.types.utils import COST_MAP_LOOKUP_KEY, ModelInfo, echoed_cost_map_fields +from litellm.types.utils import ModelInfo as ModelMapInfo +from litellm.utils import get_utc_datetime, is_generalized_model_info + +if TYPE_CHECKING: + from litellm.rust_bridge._native import ( + RouterCall, + RouterCoreHandle, + _ModelInfoSteps, # pyright: ignore[reportPrivateUsage] # the stub's step shapes + ) + +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + +# The key router-core's merge reads as an opaque value (``router-core/src/model_info.rs``). +_OPAQUE_KEY: Final = "__litellm_python_value__" + + +class _Wire: + """Values as router-core's model-info merge reads them: plain JSON, with what JSON cannot spell + (a datetime, a tuple, a non-finite float) passed as an opaque token the merge moves between + layers without reading, and restored to the same object on the way back.""" + + def __init__(self) -> None: + self._values: Final[list[object]] = [] # mutable-ok: the tokens of one query + + def to_wire(self, value: object) -> object: + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, int): + return value if -(2**63) <= value < 2**64 else self._token(value) + if isinstance(value, float): + return value if math.isfinite(value) else self._token(value) + if isinstance(value, list): + return [self.to_wire(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {str(key): self.to_wire(item) for key, item in value.items()} + return self._token(value) + + def from_wire(self, value: object) -> object: + if isinstance(value, dict): + if len(value) == 1 and _OPAQUE_KEY in value: + return self._values[int(value[_OPAQUE_KEY])] + return {str(key): self.from_wire(item) for key, item in value.items()} + if isinstance(value, list): + return [self.from_wire(item) for item in value] + return value + + def _token(self, value: object) -> Mapping[str, int]: + self._values.append(value) + return {_OPAQUE_KEY: len(self._values) - 1} + + +def _drive_model_info( + call: "RouterCall[_ModelInfoSteps]", answer: Callable[[Mapping[str, object]], object] +) -> Mapping[str, object]: + """Drive one model-info merge to its end, answering each lookup with ``answer``. A lookup that + raises raises out of the query, as it raised in the Python router.""" + outcome: object = None # rebind-ok: each lookup's answer + while True: + step = call.next(outcome) + if step["kind"] == "done": + return step + if step["kind"] == "failed": + if str(step.get("error_kind")) == "model_info": + exception: object = getattr(builtins, str(step.get("error_exception")), ValueError) + raise (exception if isinstance(exception, type) else ValueError)(step.get("error_detail")) + raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}") + outcome = answer(step) + + +def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, object]: + return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) + + +def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: + """``_operator_max_input_tokens``: the deployment's own limit, skipping one a saved + ``/model/info`` response copied from the cost map.""" + model_info: Final = _configured_model_info(deployment) + catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY) + catalog_entry: Final = ( + GetModelCostMap.loaded_model_cost_map().get(catalog_key) if isinstance(catalog_key, str) else None + ) + if catalog_entry is not None and "max_input_tokens" in echoed_cost_map_fields(model_info, catalog_entry): + return None + return coerce_token_limit(model_info.get("max_input_tokens")) + + +# The ModelGroupInfo fields router-core's group merge compares against, as the model validated them. +_GROUP_COMPARED_FIELDS: Final = frozenset( + { + "providers", + "max_input_tokens", + "max_output_tokens", + "input_cost_per_token", + "output_cost_per_token", + "supports_fast_mode", + "supported_endpoints", + "supported_reasoning_efforts", + } +) +# The ModelGroupInfo fields the Python router assigns tuples to. +_GROUP_TUPLE_FIELDS: Final = frozenset({"supported_endpoints", "supported_reasoning_efforts"}) + + +def _group_field(field: str, value: object) -> object: + """A merged group field as the Python router assigns it: a tuple field takes the list as a tuple.""" + if field in _GROUP_TUPLE_FIELDS and isinstance(value, list): + return tuple(cast(list[object], value)) # cast-ok: the merge's list + return value + + +@dataclasses.dataclass +class _GroupBuild: + """One group-info merge as the host answers it: the deployments it reads, each one's validated + params once its first lookup builds them, and the group built from the first deployment.""" + + model_list: "Sequence[dict[str, Any]]" # mutable-ok: the stored entries, read only + team_id: str | None + # Filled as the lookups arrive. + params: dict[int, LiteLLM_Params] = dataclasses.field(default_factory=dict) # mutable-ok: see above + info: ModelGroupInfo | None = None + + +class HostModelInfo: + """Mixed into ``HostRegistry``, which holds the model list, the indexes and the host.""" + + def _live_handle(self) -> "RouterCoreHandle | None": ... # the registry's: the handle ``RustCore`` set + + def _core_handle(self) -> "RouterCoreHandle": + handle: Final = self._live_handle() + if handle is None: + raise RuntimeError("the host registry reads model info through its Rust core") + return handle + + def _core_get_model_list_of_routed_group( + self, model_group: str, team_id: str | None = None + ) -> list: # mutable-ok: a fresh list, as the core returns it + if team_id is not None and self._team_deployments_named(team_id, model_group): + return self._core_get_all_deployments(model_name=model_group, team_id=team_id) + named: Final = [ + *self._core_get_all_deployments(model_name=model_group), + *self._core_get_model_list_from_routing_groups(model_name=model_group), + ] + return named or self._get_wildcard_deployments(model_name=model_group) + + def _get_wildcard_deployments( + self, model_name: str, team_id: str | None = None + ) -> list[dict[str, object]]: # mutable-ok: a fresh list + """The deployments a wildcard pattern routes ``model_name`` to, renamed to it (the routing + runs in router-core, through the registry's pattern router views).""" + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + matches: Final = [ + *self._core_pattern_router.get_deployments_by_pattern(model=model_name), + *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), + ] + return [{**DeploymentTypedDict(**m), "model_name": model_name} for m in matches] # pyright: ignore[reportCallIssue] # a stored entry + + def _core_get_model_group_info( + self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None + ) -> ModelGroupInfo | None: + """ + For a given model group name, return the combined model info + + Returns: + - ModelGroupInfo if able to construct a model group + - None if error constructing model group info or hidden model group (unless include_hidden) + """ + ## Check if model group alias + if model_group in self._core_model_group_alias: + item: Final = self._core_model_group_alias[model_group] + if isinstance(item, str): + _router_model_group = item # rebind-ok: ported from the Python core unchanged + elif isinstance(item, dict): + if item["hidden"] is True and not include_hidden: + return None + else: + _router_model_group = item["model"] # rebind-ok: ported from the Python core unchanged + else: + return None + + return self._set_model_group_info( + model_group=_router_model_group, + user_facing_model_group_name=model_group, + team_id=team_id, + ) + + ## Check if actual model + return self._set_model_group_info( + model_group=model_group, user_facing_model_group_name=model_group, team_id=team_id + ) + + def _set_model_group_info( + self, model_group: str, user_facing_model_group_name: str, team_id: str | None = None + ) -> ModelGroupInfo | None: + """ + For a given model group name, return the combined model info + + Returns: + - ModelGroupInfo if able to construct a model group + - None if error constructing model group info + + router-core merges the group (``router-core/src/group_info.rs``); this picks its + deployments, answers what needs this process (validated params, model info, provider, + capabilities), builds ``ModelGroupInfo`` from the first deployment and sets what the merge + assigned on it. + """ + serves_team_deployments: Final = team_id is not None and bool( + self._team_deployments_named(team_id, model_group) + ) + model_list: Final = self._core_get_model_list_of_routed_group(model_group, team_id) + description: Final[str | None] = next( + (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), + None, + ) + # The wildcard check is asked only for a deployment that is not the group's by name. + pattern_routes: Final = ( + not serves_team_deployments + and any("model_name" in m and m["model_name"] != model_group for m in model_list) + and self._core_pattern_router.route(model_group) is not None + ) + wire: Final = _Wire() + group: Final = _GroupBuild(model_list=model_list, team_id=team_id) + call: Final = self._core_handle().model_group_info( + { + "model_group": model_group, + "user_facing_model_group_name": user_facing_model_group_name, + "serves_team_deployments": serves_team_deployments, + "pattern_routes": pattern_routes, + "description": description, + "deployments": [wire.to_wire(model) for model in model_list], + } + ) + done: Final = _drive_model_info(call, partial(self._answer_group_lookup, group=group, wire=wire)) + answer: Final = as_str_mapping(done["value"]) + if answer is None or group.info is None: + return None + for field, value in (as_str_mapping(answer["assignments"]) or {}).items(): + setattr(group.info, field, _group_field(field, wire.from_wire(value))) + return group.info + + def _answer_group_lookup(self, step: Mapping[str, object], group: "_GroupBuild", wire: "_Wire") -> object: + """One lookup of a group-info merge: a deployment's facts, or the group's first entry.""" + match step["lookup"]: + case "group_params": + index: Final = int(str(step["index"])) + litellm_params: Final = LiteLLM_Params(**group.model_list[index]["litellm_params"]) + group.params[index] = litellm_params # rebind-ok: the merge's state, which each lookup advances + return { + "model": litellm_params.model, + "configurable_clientside_auth_params": wire.to_wire( + litellm_params.configurable_clientside_auth_params + ), + } + case "group_deployment": + return self._group_deployment(group, int(str(step["index"])), wire) + case "group_capabilities": + deployment: Final = group.model_list[int(str(step["index"]))] + merged_info: Final[object] = wire.from_wire(step["model_info"]) + model_info: Final = cast(Mapping[str, object], merged_info) # cast-ok: the merged model info + litellm_model: Final = str(step["litellm_model"]) + llm_provider: Final = str(step["llm_provider"]) + efforts: Final = resolve_supported_reasoning_efforts( + model_info, deployment_is_mapped=step["mapped"] is True + ) + return { + "supported_endpoints": wire.to_wire( + list( + deployment_supported_endpoints( + model_info.get("supported_endpoints"), deployment, llm_provider + ) + ) # pyright: ignore[reportArgumentType] # a stored entry, as the core passes it + if team_may_use_deployment(_configured_model_info(deployment).get("team_id"), group.team_id) # pyright: ignore[reportArgumentType] # the configured team id + else [] + ), + "supports_fast_mode": AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider), + "reasoning_efforts": None if efforts is None else list(efforts), + } + case "group_seed": + entry: Final[object] = wire.from_wire(step["info"]) + seed: Final = cast(dict[str, Any], entry) # cast-ok: the first deployment's entry + group.info = ModelGroupInfo(**seed) # rebind-ok: the merge's state, built from the first deployment + return group.info.model_dump(mode="json", include=set(_GROUP_COMPARED_FIELDS)) + case "reasoning_intersect": + current: Final = cast("Sequence[str] | None", step["current"]) # cast-ok: plain JSON efforts + resolved: Final = cast("Sequence[str] | None", step["resolved"]) # cast-ok: plain JSON efforts + intersected: Final = intersect_supported_reasoning_efforts(current, resolved) + return None if intersected is None else list(intersected) + case _: + return self._answer_model_info_lookup(step, wire) + + def _group_deployment(self, group: "_GroupBuild", index: int, wire: "_Wire") -> Mapping[str, object]: + """A group deployment's model info (None when it fails), whether the catalog maps it, and its + provider; with no model info, the provider's supported params.""" + model: Final = group.model_list[index] + litellm_params: Final = group.params[index] + model_info_dict: Final = model.get("model_info", {}) + model_info: object + try: + model_id: Final = model_info_dict.get("id", None) + model_info = ( # rebind-ok: one branch binds it + self._core_get_deployment_model_info(model_id=model_id, model_name=litellm_params.model) + if model_id is not None + else None + ) + except Exception: # noqa: BLE001 # ported as the core writes it + model_info = None # rebind-ok: see above + deployment_is_mapped: Final = deployment_is_catalog_mapped(model_info, model_info_dict) # pyright: ignore[reportArgumentType] # the merged model info, or None + litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) + answer: Final[dict[str, object]] = { # mutable-ok: built here + "model_info": wire.to_wire(model_info), + "mapped": deployment_is_mapped, + "litellm_model": litellm_model, + "llm_provider": llm_provider, + } + if model_info is None: + answer["supported_openai_params"] = wire.to_wire( + self._host.invoke("get_supported_openai_params")(model=litellm_model, custom_llm_provider=llm_provider) + ) + return answer + + def _cached_get_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + """ + Cached version of get_model_group_info, uses @lru_cache wrapper + + This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request + """ + return self._core_get_model_group_info(model_group) + + def _core_cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + return self._cached_get_model_group_info(model_group) + + @overload + def _core_get_router_model_info( + self, + deployment: Union[dict, "Deployment"], # mutable-ok: ported from the Python core unchanged + received_model_name: str, + id: None = None, + ) -> ModelMapInfo: + pass + + @overload + def _core_get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> ModelMapInfo: + pass + + def _core_get_router_model_info( + self, + deployment: Union[dict, "Deployment"] | None, # mutable-ok: ported from the Python core unchanged + received_model_name: str, + id: str | None = None, + ) -> ModelMapInfo: + """ + For a given model id, return the model info (max tokens, input cost, output cost, etc.). + + Augment litellm info with additional params set in `model_info`. + + For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set. + For every other provider, a base_model naming the same model as `model:` can only widen its context window. + + router-core merges it (``router-core/src/model_info.rs``); this checks the deployment and + answers the cost-map lookups the merge asks for while it runs. + + Returns + - ModelInfo - If found -> typed dict with max tokens, input cost, etc. + + Raises: + - ValueError -> If model is not mapped yet + """ + if id is not None: + _deployment: Final = self._core_get_deployment(model_id=id) + if _deployment is not None: + deployment = _deployment # rebind-ok: ported from the Python core unchanged + + if deployment is None: + raise ValueError("Deployment not found") + + model_info_base_model: Final = (deployment.get("model_info") or {}).get("base_model", None) + params_base_model: Final = ( + (deployment.get("litellm_params") or {}).get("base_model", None) if model_info_base_model is None else None + ) + + ## GET PROVIDER - reuse LiteLLM_Params if already constructed + litellm_params_data: Final = deployment.get("litellm_params") + litellm_params: LiteLLM_Params + if isinstance(litellm_params_data, LiteLLM_Params): + litellm_params = litellm_params_data # rebind-ok: ported from the Python core unchanged + elif isinstance(litellm_params_data, dict) and "model" in litellm_params_data: + litellm_params = LiteLLM_Params.model_validate( # rebind-ok: ported from the Python core unchanged + litellm_params_data + ) + else: + raise ValueError( + f"Deployment missing valid litellm_params. " + f"Got: {type(litellm_params_data).__name__}, " + f"deployment_id: {(deployment.get('model_info') or {}).get('id', 'unknown')}" + ) + + wire: Final = _Wire() + user_model_info: dict[str, object] = {} # mutable-ok: filled when the merge asks for it # rebind-ok: see above + + def read_user_model_info() -> object: + nonlocal user_model_info + raw_user_model_info: Final = deployment.get("model_info") + user_model_info = dict( # rebind-ok: see above + _MODEL_INFO_ADAPTER.validate_python( + raw_user_model_info.model_dump(exclude_none=True) + if isinstance(raw_user_model_info, BaseModel) + else raw_user_model_info or {} + ) + ) + return wire.to_wire(user_model_info) + + def answer(step: Mapping[str, object]) -> object: + if step["lookup"] == "provider": + _model, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( + model=litellm_params.model, + litellm_params=litellm_params, + ) + return [_model, custom_llm_provider] + if step["lookup"] == "user_model_info": + return read_user_model_info() + return self._answer_model_info_lookup(step, wire) + + query: Final = { + "deployment_id": wire.to_wire((deployment.get("model_info") or {}).get("id")), + "model_info_base_model": wire.to_wire(model_info_base_model), + "params_base_model": wire.to_wire(params_base_model), + "params_model": litellm_params.model, + "extra_base_model": wire.to_wire((litellm_params.model_extra or {}).get("base_model")), + "received_model_name": received_model_name, + } + done: Final = _drive_model_info(self._core_handle().router_model_info(query), answer) + merged: Final = wire.from_wire(done["info"]) + if not isinstance(merged, dict): + return merged # pyright: ignore[reportReturnType] # the core returns get_model_info's answer as is + # The deployment's own values are its objects, as the Python router merges them. + for key in cast(list[str], done["user_keys"]): # cast-ok: router-core's key list + merged[key] = user_model_info[key] + return merged # pyright: ignore[reportReturnType] # the merged map, as the core builds it + + def _core_get_deployment_model_info(self, model_id: str, model_name: str) -> ModelInfo | None: + """ + For a given model id, return the model info + + 1. Check if model_id is in model info + 2. If not, check if litellm model name is in model info + 3. If not, return None + + router-core merges it (``router-core/src/model_info.rs``); this answers its lookups. + """ + wire: Final = _Wire() + call: Final = self._core_handle().deployment_model_info(model_id, wire.to_wire(model_name)) + done: Final = _drive_model_info(call, partial(self._answer_model_info_lookup, wire=wire)) + return wire.from_wire(done["info"]) # pyright: ignore[reportReturnType] # the merged map, as the core builds it + + def _answer_model_info_lookup(self, step: Mapping[str, object], wire: "_Wire") -> object: + """One cost-map lookup of a model-info merge, answered from this process.""" + match step["lookup"]: + case "model_info": + return self._model_info_row(wire.from_wire(step["model"]), wire, step) + case "cost_entry": + key: Final = str(step["key"]) + model_cost: Final = litellm.model_cost if step["process"] else self._host.dynamic().model_cost + return {"entry": wire.to_wire(model_cost.get(key))} if key in model_cost else None + case "pattern_models": + return self._pattern_models(str(step["name"]), wire) + case "bedrock_base": + return get_bedrock_base_model(str(step["model_id"])) + case "discovered": + return wire.to_wire(dict(self._core_get_discovered_model_info(wire.from_wire(step["id"])))) # pyright: ignore[reportArgumentType] # the deployment's id, as configured + case "log": + log: Final = verbose_router_logger.error if step["level"] == "error" else verbose_router_logger.debug + log("%s", step["message"]) + return None + case other: + raise ValueError(f"router-core asked an unknown model-info lookup {other!r}") + + def _model_info_row(self, model: object, wire: "_Wire", step: Mapping[str, object]) -> object: + """``get_model_info(model=...)`` as a lookup answers it: the row, ``{"failed": True}`` when a + tolerated lookup raises, and for an exact lookup whether a generalization rule made it.""" + # The base model's own window is read through the process host, as the Python router reads it. + host: Final = python_host if step["exact"] else self._host + try: + row: Final = host.invoke("get_model_info")(model=model) + except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model + if step["tolerate"]: + return {"failed": True} + raise + if step["exact"]: + return {"row": wire.to_wire(row), "generalized": is_generalized_model_info(row)} + return wire.to_wire(row) + + def _pattern_models(self, name: str, wire: "_Wire") -> object: + """The ``{id, model}`` of each deployment a wildcard pattern routes ``name`` to.""" + potential_models: Final = self._core_pattern_router.route(name) + if potential_models is None: + return None + routed: Final[list[object]] = [] # mutable-ok: built here + for potential_model in potential_models: + try: + routed.append( + { + "id": wire.to_wire((potential_model.get("model_info") or {}).get("id")), + "model": wire.to_wire((potential_model.get("litellm_params") or {}).get("model")), + } + ) + except Exception: # noqa: BLE001 # ported as the core writes it + verbose_router_logger.debug("model info lookup failed; ignored as the core ignores it") + return routed + + def _core_get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) + if ( + model_id is not None + and isinstance(cached, DiscoveredDeploymentModelInfo) + and cached.deployment is self._core_get_model_info(model_id) + ): + configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) + return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) + return MappingProxyType({}) + + def _core_get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: + """ + Return what the concrete deployments behind model_name contribute to its + /v1/models entry: the cost-map keys for their underlying models, plus the widest + configured or discovered token limits. Resolved via O(1) index + lookup. + + Returns None for wildcard-expanded or unknown names, where the listed name is the + real model name and no deployment-specific information exists, and treats a + malformed configured limit as absent rather than failing the listing. + + The whole group is read rather than just its first deployment, so a group that + mixes models does not advertise a window that depends on config order; the widest + one is reported, which is what get_model_group_info already shows the Admin UI. + Keys are deduplicated, so the ordinary group of interchangeable deployments of one + model still costs the caller a single cost-map lookup. Unlike get_model_group_info, + this never triggers pattern matching or deep copies, so it is safe to call per + listed model on the /v1/models hot path. + + router-core combines the deployments (``router-core/src/group_info.rs``). + """ + deployments: Final = self._core_deployments_named(model_name) + if not deployments: + return None + wire: Final = _Wire() + call: Final = self._core_handle().model_listing_info([wire.to_wire(deployment) for deployment in deployments]) + done: Final = _drive_model_info(call, partial(self._answer_model_info_lookup, wire=wire)) + listing: Final = as_str_mapping(done["value"]) or {} + return DeploymentModelListingInfo( + cost_map_keys=tuple(cast(list[str], listing["cost_map_keys"])), # cast-ok: router-core's key list + max_input_tokens=cast("int | None", listing["max_input_tokens"]), # cast-ok: router-core's limit + max_output_tokens=cast("int | None", listing["max_output_tokens"]), # cast-ok: router-core's limit + ) + + def _core_get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": + """ + Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete + deployment of model_name, via O(1) index lookup. + + Returns (None, None) for wildcard-expanded or unknown names, and treats a + malformed configured value as absent rather than failing the caller. + + Deliberately reads one deployment rather than aggregating the group the way + get_model_listing_info does: its caller truncates an embedding input to this + value, so the widest window in a mixed group would be the wrong answer there. + """ + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return (None, None) + wire: Final = _Wire() + configured: Final = { + key: wire.to_wire(value) for key, value in deployment.model_info.model_dump(exclude_none=True).items() + } + call: Final = self._core_handle().configured_token_limits(wire.to_wire(deployment.model_info.id), configured) + done: Final = _drive_model_info(call, partial(self._answer_model_info_lookup, wire=wire)) + limits: Final = cast("list[int | None]", done["value"]) # cast-ok: router-core's two limits + return (limits[0], limits[1]) + + def _core_get_configured_mode(self, model_name: str) -> "str | None": + """Return the mode explicitly configured for a concrete deployment.""" + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + return self._core_handle().configured_text(_Wire().to_wire(deployment.model_info.get("mode"))) + + def _core_get_configured_display_name(self, model_name: str) -> "str | None": + """ + Return the display_name explicitly configured in a concrete deployment's + model_info for model_name, via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, and treats a + non-string or empty configured value as absent rather than failing the + listing. Like get_configured_token_limits, this never triggers pattern + matching or deep copies, so it is safe to call per listed model on the + /v1/models hot path. + """ + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + return self._core_handle().configured_text(_Wire().to_wire(deployment.model_info.get("display_name"))) + + def _core_get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: + """ + Return the service_tiers value each deployment a request from team_id + to model_name can route to (see _routable_deployments) configures in + its model_info, unvalidated and in model_list order, None for a + deployment that sets none; the caller validates the shape it expects + and decides how the deployments combine. + + Returns an empty tuple for wildcard-expanded or unknown names. + """ + wire: Final = _Wire() + tiers: Final = self._core_handle().configured_service_tiers( + [ + wire.to_wire(dict(_configured_model_info(deployment))) + for deployment in self._routable_deployments(model_name, team_id) + ] + ) + return tuple(wire.from_wire(tier) for tier in tiers) + + def _core_get_configured_max_input_tokens( + self, model_name: str, team_id: str | None = None + ) -> tuple[int | None, ...]: + """ + Return the max_input_tokens each deployment a request from team_id to + model_name can route to (see _routable_deployments) sets in its own + model_info, in model_list order, None for a deployment that sets none + or a malformed one. Unlike get_model_listing_info, neither the cost map + nor a discovered limit is read, so only an operator's own value counts, + and a value a saved /model/info response copied from the cost map is + skipped too. + + Returns an empty tuple for wildcard-expanded or unknown names. + """ + return tuple( + _operator_max_input_tokens(deployment) for deployment in self._routable_deployments(model_name, team_id) + ) + + async def _core_get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: + """ + Returns current tpm/rpm usage for model group + + Parameters: + - model_group: str - the received model name from the user (can be a wildcard route). + + Returns: + - usage: Tuple[tpm, rpm] + """ + dt: Final = get_utc_datetime() + current_minute: Final = dt.strftime("%H-%M") # use the same timezone regardless of system clock + tpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + rpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + + model_list: Final = self._core_get_model_list(model_name=model_group) + if model_list is None: # no matching deployments + return None, None + + for model in model_list: + id: str | None = model.get("model_info", {}).get("id") + litellm_model: str | None = model["litellm_params"].get( + "model" + ) # USE THE MODEL SENT TO litellm.completion() - consistent with how global_router cache is written. + if id is None or litellm_model is None: + continue + tpm_keys.append(self._core_handle().usage_key(id, litellm_model, "tpm", current_minute)) + rpm_keys.append(self._core_handle().usage_key(id, litellm_model, "rpm", current_minute)) + combined_tpm_rpm_keys: Final = tpm_keys + rpm_keys + + with service_target(ROUTER_USAGE_TARGET): + combined_tpm_rpm_values: Final = await self._core_cache.async_batch_get_cache(keys=combined_tpm_rpm_keys) + if combined_tpm_rpm_values is None: + return None, None + + rpm_start: Final = len(tpm_keys) + tpm_usage_list: Final[list | None] = combined_tpm_rpm_values[:rpm_start] # mutable-ok: ported from the Python core unchanged # fmt: skip + rpm_usage_list: Final[list | None] = combined_tpm_rpm_values[rpm_start:] # mutable-ok: ported from the Python core unchanged # fmt: skip + + ## TPM + tpm_usage: int | None = None # rebind-ok: ported from the Python core unchanged + if tpm_usage_list is not None: + for t in tpm_usage_list: + if isinstance(t, int): + if tpm_usage is None: + tpm_usage = 0 + tpm_usage += t + ## RPM + rpm_usage: int | None = None # rebind-ok: ported from the Python core unchanged + if rpm_usage_list is not None: + for t in rpm_usage_list: + if isinstance(t, int): + if rpm_usage is None: + rpm_usage = 0 + rpm_usage += t + return tpm_usage, rpm_usage + + async def get_model_group_io_token_usage(self, model_group: str) -> tuple[int | None, int | None]: + """ + Returns current ITPM/OTPM usage for a model group (sum across deployments). + """ + dt: Final = get_utc_datetime() + current_minute: Final = dt.strftime("%H-%M") + itpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + otpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + + model_list: Final = self._core_get_model_list(model_name=model_group) + if model_list is None: + return None, None + + for model in model_list: + model_id: str | None = model.get("model_info", {}).get("id") + litellm_model: str | None = model["litellm_params"].get("model") + if model_id is None or litellm_model is None: + continue + itpm_keys.append(self._core_handle().usage_key(model_id, litellm_model, "itpm", current_minute)) + otpm_keys.append(self._core_handle().usage_key(model_id, litellm_model, "otpm", current_minute)) + + with service_target(ROUTER_USAGE_TARGET): + combined_values: Final = await self._core_cache.async_batch_get_cache(keys=itpm_keys + otpm_keys) + if combined_values is None: + return None, None + + itpm_values: Final = combined_values[: len(itpm_keys)] + otpm_values: Final = combined_values[len(itpm_keys) :] + + total_itpm: int | None = None # rebind-ok: ported from the Python core unchanged + for value in itpm_values: + if isinstance(value, int): + total_itpm = (total_itpm or 0) + value + + total_otpm: int | None = None # rebind-ok: ported from the Python core unchanged + for value in otpm_values: + if isinstance(value, int): + total_otpm = (total_otpm or 0) + value + + return total_itpm, total_otpm + + async def _core_get_remaining_model_group_usage( + self, model_group: str + ) -> dict[str, int]: # mutable-ok: ported from the Python core unchanged + model_group_info: Final = self._cached_get_model_group_info(model_group) + + returned_dict: Final[dict[str, int]] = {} # mutable-ok: ported from the Python core unchanged + + # ITPM/OTPM groups emit input/output token headers, but they may also set + # tpm/rpm, so build both sets rather than returning early - clients and + # prometheus gauges that read the standard headers still get data. + if model_group_info is not None and (model_group_info.itpm is not None or model_group_info.otpm is not None): + current_itpm, current_otpm = await self.get_model_group_io_token_usage(model_group) + returned_dict.update( + build_io_token_rate_limit_headers( + itpm_limit=model_group_info.itpm, + otpm_limit=model_group_info.otpm, + current_itpm=current_itpm, + current_otpm=current_otpm, + ) + ) + + tpm_limit: Final = model_group_info.tpm if model_group_info is not None else None + rpm_limit: Final = model_group_info.rpm if model_group_info is not None else None + + if tpm_limit is None and rpm_limit is None: + return returned_dict + + current_tpm, current_rpm = await self._core_get_model_group_usage(model_group) + + if tpm_limit is not None: + returned_dict["x-ratelimit-remaining-tokens"] = tpm_limit - (current_tpm or 0) + returned_dict["x-ratelimit-limit-tokens"] = tpm_limit + if rpm_limit is not None: + returned_dict["x-ratelimit-remaining-requests"] = rpm_limit - (current_rpm or 0) + returned_dict["x-ratelimit-limit-requests"] = rpm_limit + + return returned_dict + + async def set_response_headers( + self, + response: object, + model_group: str | None = None, + request_kwargs: dict[str, object] | None = None, # mutable-ok: ported from the Python core unchanged + ) -> Any: # noqa: ANN401 # the core's signature, ported unchanged + """ + Add the most accurate rate limit headers for a given model response. + + ## TODO: add model group rate limit headers + # - if healthy_deployments > 1, return model group rate limit headers + # - else return the model's rate limit headers + """ + response = prepare_response_for_header_attachment(response) # rebind-ok: ported from the Python core unchanged + if response is None: + return response + + additional_headers: Final = ensure_response_additional_headers(response) + apply_response_model_id( + response, find_deployment_metadata(request_kwargs) if request_kwargs is not None else None + ) + additional_headers["x-litellm-model-group"] = model_group + apply_quality_router_decision_headers(additional_headers, request_kwargs) + additional_headers.update(complexity_router_decision_headers(request_kwargs)) + + if model_group is not None: + remaining_usage: Final = await self._core_get_remaining_model_group_usage(model_group) + apply_remaining_usage_headers(additional_headers, remaining_usage) + return response diff --git a/litellm/router_rust/surface/model_list.py b/litellm/router_rust/surface/model_list.py new file mode 100644 index 00000000000..405a4b47d7b --- /dev/null +++ b/litellm/router_rust/surface/model_list.py @@ -0,0 +1,375 @@ +""" +``HostedModelList``: the Rust core's live ``model_list``, read back from router-core. + +router-core holds the deployments. This list is what ``registry().all()`` and the deprecated +``model_list`` attribute hand out: a ``list`` (so callers that check for one, serialize it or +validate it keep working) whose contents are router-core's deployment records, read through the +handle after every write. Every write to it is a changeset instead of a Python-side mutation: + +- ``append`` / ``extend`` / an ``insert`` at the end add at the end (``append``) +- removing one deployment whose id no other entry shares is a ``delete`` +- any other write (an assignment, a slice, a removal of one of several entries sharing an id, + ``clear``, ``sort``, ...) hands router-core the resulting list (``replace``) + +A build serves the deployments it added once it ends (or fails), and the list reads them back +then. After writes that only appended, it reads just the appended records +(``RouterCoreHandle.deployments_from``), so adding deployments one at a time costs each one its own +record rather than the whole list. These writes are the raw ones a caller makes: like the Python +router's list, they reindex nothing. + +An entry read back from router-core is a dict whose writes (its own, or a nested dict's) hand +router-core the list again, so an edit in place reaches it as the Python router's list would show +it. Each deployment keeps one Python object while router-core's record for it is unchanged, keyed by +its id and its place among entries sharing that id, so identity checks on stored entries (the +model info caches) mean what they did: the same object until a write changes that deployment. +""" + +import copy +import datetime +from collections.abc import Callable, Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + SupportsIndex, + TypeAlias, + TypeVar, + cast, # noqa: TID251 # any value a caller wrote into the list + overload, +) + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.types.router import ModelAccessWindow + +if TYPE_CHECKING: + from litellm.rust_bridge._native import RouterCoreHandle + +Entry: TypeAlias = dict[str, object] +# (id, its place among the entries sharing it) -> (the record it was read as, the entry object) +_Stable: TypeAlias = dict[tuple[str, int], tuple[object, Entry]] + + +_DATETIME: Final = TypeAdapter(datetime.datetime) +_Field = TypeVar("_Field") # rebind-ok: a TypeVar cannot be Final + + +def _dumped_as(adapter: TypeAdapter[_Field]) -> Callable[[object], object]: + """A value validated by ``adapter``, then dumped back to its Python form.""" + + def dump(value: object) -> object: + return adapter.dump_python(adapter.validate_python(value), exclude_none=True) + + return dump + + +# The ``ModelInfo`` fields whose Python value is no JSON value, which router-core holds as strings. +_TYPED_MODEL_INFO: Final[Mapping[str, Callable[[object], object]]] = MappingProxyType( + { + "created_at": _dumped_as(_DATETIME), + "updated_at": _dumped_as(_DATETIME), + "ptu_effective_from": _dumped_as(_DATETIME), + "ptu_effective_to": _dumped_as(_DATETIME), + "access_windows": _dumped_as(TypeAdapter(tuple[ModelAccessWindow, ...])), + } +) + + +def typed_deployment(record: Mapping[str, object]) -> Mapping[str, object]: + """A deployment record as ``Deployment.to_json`` dumped it: its ``model_info`` typed again. One + with none of those fields comes back as it is, and so does a value a raw write left unreadable.""" + info: Final = as_str_mapping(record.get("model_info")) + if info is None or info.keys().isdisjoint(_TYPED_MODEL_INFO): + return record + typed: Final[dict[str, object]] = {} # mutable-ok: filled here + for key, dumped in _TYPED_MODEL_INFO.items(): + value = info.get(key) + if value is None: + continue + try: + typed[key] = dumped(value) + except ValidationError: + continue + return {**record, "model_info": {**info, **typed}} + + +def _record(value: object) -> Mapping[str, object]: + return cast(Mapping[str, object], value) # cast-ok: a JSON object router-core wrote, or a dict inside one + + +def _entry_id(entry: object) -> str | None: + info: Final = as_str_mapping((as_str_mapping(entry) or {}).get("model_info")) or {} + value: Final = info.get("id") + return None if value is None else str(value) + + +def _keys( + entries: Iterable[object], seen: Mapping[str, int] = MappingProxyType({}) +) -> tuple[list[tuple[str, int] | None], Mapping[str, int]]: # mutable-ok: a fresh list + """Each entry's identity key (its id and how many earlier entries share it, ``seen`` counting + the ids of entries before these), and the counts once past them.""" + counts: Final[dict[str, int]] = dict(seen) # mutable-ok: counted here + keys: Final[list[tuple[str, int] | None]] = [] # mutable-ok: filled here + for entry in entries: + entry_id = _entry_id(entry) + if entry_id is None: + keys.append(None) + continue + nth = counts.get(entry_id, 0) + counts[entry_id] = nth + 1 + keys.append((entry_id, nth)) + return keys, counts + + +class _HostedEntry(dict[str, object]): + """A deployment entry read back from router-core: a write to it, or to a dict inside it, hands + router-core the list again (``replace``), so it learns what the entry now says. A copy is a + plain dict.""" + + def __init__(self, record: Mapping[str, object], written: Callable[[], None]) -> None: + super().__init__( + (key, value if (nested := as_str_mapping(value)) is None else _HostedEntry(nested, written)) + for key, value in record.items() + ) + self._written: Final = written + + def __setitem__(self, key: str, value: object) -> None: + super().__setitem__(key, value) + self._written() + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + self._written() + + def pop(self, key: str, *default: object) -> object: + value: Final = super().pop(key, *default) + self._written() + return value + + def popitem(self) -> tuple[str, object]: + item: Final = super().popitem() + self._written() + return item + + def clear(self) -> None: + super().clear() + self._written() + + def update(self, *args: object, **kwargs: object) -> None: # kwargs-ok: dict.update's own arguments + super().update(*args, **kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # dict.update's own arguments + self._written() + + def setdefault(self, key: str, default: object = None) -> object: + if key in self: + return self[key] + self[key] = default + return default + + def __ior__(self, other: object) -> "_HostedEntry": # pyright: ignore[reportIncompatibleMethodOverride] # dict's in-place merge, as a write + self.update(other) + return self + + def __copy__(self) -> dict[str, object]: # mutable-ok: a plain copy + return dict(self) + + def __deepcopy__(self, memo: dict[int, object]) -> dict[str, object]: # mutable-ok: a plain copy + return copy.deepcopy(dict(self), memo) + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (dict, (copy.deepcopy(dict(self)),)) + + +class HostedModelList(list[Entry]): + """The live ``model_list``: router-core's deployments, every write a changeset.""" + + def __init__( + self, + handle: Callable[[], "RouterCoreHandle | None"], + apply: Callable[[Mapping[str, object]], None], + ) -> None: + super().__init__() + self._handle: Final = handle + self._apply: Final = apply + # The handle and registry version the contents were last read at. + self._read_at: tuple[object, int] | None = None + self._stable: _Stable = {} + # How many of the shown entries hold each id, for the keys of entries appended next. + self._id_counts: Mapping[str, int] = {} + # While a build runs: the length, id counts and read to go back to when it ends. + self._building: tuple[int, Mapping[str, int], tuple[object, int] | None] | None = None + + # -- reading ---------------------------------------------------------------------------------- + + def refresh(self) -> None: + """Read router-core's deployments again when its registry moved since the last read.""" + handle: Final = self._handle() + if handle is None or self._building is not None: + # While a build runs, the list shows what it accepted until router-core serves it. + return + version: Final = handle.registry_version() + if self._read_at is not None and self._read_at[0] is handle and self._read_at[1] == version: + return + read_at: Final = self._read_at + self._read_at = (handle, version) + if read_at is not None and read_at[0] is handle and handle.registry_rewritten() <= read_at[1]: + # Only appends since the last read: the shown entries stand, and the new ones follow. + super().extend(self._kept(handle.deployments_from(len(self)), appended=True)) + return + self._show(self._kept(handle.deployments())) + + def _kept(self, records: Sequence[object], *, appended: bool = False) -> list[Entry]: # mutable-ok: a fresh list + """``records`` as entries, each the object it was while its record is unchanged; when + ``appended``, they follow the shown entries, which keep theirs.""" + stable: Final[_Stable] = self._stable if appended else {} + keys, counts = _keys(records, self._id_counts if appended else {}) + entries: Final[list[Entry]] = [] # mutable-ok: filled here + for key, record in zip(keys, records, strict=True): + typed = typed_deployment(_record(record)) + held = self._stable.get(key) if key is not None else None + kept = held is not None and held[0] == typed + entry = held[1] if held is not None and kept else _HostedEntry(typed, self._entry_written) + if key is not None: + stable[key] = (held[0] if held is not None and kept else copy.deepcopy(typed), entry) + entries.append(entry) + self._stable = stable + self._id_counts = counts + return entries + + # -- a build ---------------------------------------------------------------------------------- + + def begin_build(self, *, replacing: bool) -> None: + """A build starts: router-core serves what it adds once it ends, so until then the list + shows each deployment it accepts here (``built``), after the entries it keeps, or alone + when it replaces them, as the Python router's list grows while it adds them.""" + self.refresh() + if replacing: + self._show([]) + self._id_counts = {} + self._building = (len(self), self._id_counts, self._read_at) + + def built(self, entry: Mapping[str, object]) -> None: + """A deployment the build accepted, shown until the build ends.""" + if self._building is not None: + super().extend(self._kept([entry], appended=True)) + + def end_build(self) -> None: + """The build ended (or failed): router-core serves what it added, read back in place of + the entries shown meanwhile, each kept as the object it was shown as while unchanged.""" + if self._building is None: + return + start, counts, read_at = self._building + self._building = None + super().__delitem__(slice(start, None)) + self._id_counts = counts + self._read_at = read_at + self.refresh() + + def _seed(self, entries: Sequence[object]) -> None: + """Keep ``entries``, the list router-core now holds, as the objects its records for them + read back as.""" + for key, entry in zip(_keys(entries)[0], entries, strict=True): + self._seed_one(key, entry) + + def _seed_appended(self, entries: Sequence[object]) -> None: + """Keep ``entries``, appended after the shown ones, as the objects they read back as.""" + for key, entry in zip(_keys(entries, self._id_counts)[0], entries, strict=True): + self._seed_one(key, entry) + + def _seed_one(self, key: tuple[str, int] | None, entry: object) -> None: + if key is not None and isinstance(entry, dict): + stored: Final = cast(Entry, entry) # cast-ok: a model_list entry is a dict + self._stable[key] = (copy.deepcopy(stored), stored) + + def _entry_written(self) -> None: + """A write inside an entry: router-core takes the list as it now reads.""" + self._replace(list(self)) + + def _show(self, entries: list[Entry]) -> None: # mutable-ok: the entries shown + super().__setitem__(slice(None), entries) + + # -- writing ---------------------------------------------------------------------------------- + + def _replace(self, entries: list[Entry]) -> None: # mutable-ok: the list router-core takes + # Seeded first: router-core's records for them read back as these objects however soon + # the list is read again. + self._seed(entries) + self._apply({"replace": entries}) + self.refresh() + + def replace(self, entries: Iterable[object]) -> None: + """Every deployment at once (``router.model_list = [...]``).""" + self._replace(cast(list[Entry], list(entries))) # cast-ok: the entries as assigned, which router-core validates + + def append(self, value: Entry) -> None: + self.refresh() + self._seed_appended([value]) + self._apply({"append": [value]}) + self.refresh() + + def extend(self, values: Iterable[Entry]) -> None: + for value in tuple(values): + self.append(value) + + def __iadd__(self, values: Iterable[Entry]) -> "HostedModelList": # pyright: ignore[reportIncompatibleMethodOverride] # list's in-place add, as a changeset + self.extend(values) + return self + + def insert(self, index: SupportsIndex, value: Entry) -> None: + if int(index) >= len(self): + self.append(value) + return + changed: Final = list(self) + changed.insert(index, value) + self._replace(changed) + + @overload + def __setitem__(self, index: SupportsIndex, value: Entry) -> None: ... + @overload + def __setitem__(self, index: slice, value: Iterable[Entry]) -> None: ... + def __setitem__(self, index: SupportsIndex | slice, value: object) -> None: + changed: Final = list(self) + changed[index] = value # pyright: ignore[reportArgumentType, reportCallIssue] # list's own item and slice assignment + self._replace(changed) + + def __delitem__(self, index: SupportsIndex | slice) -> None: + changed: Final = list(self) + if isinstance(index, slice): + del changed[index] + self._replace(changed) + return + removed: Final = changed.pop(index) + removed_id: Final = _entry_id(removed) + if removed_id is None or any(_entry_id(entry) == removed_id for entry in changed): + # A delete removes every deployment with the id; the list removes only this one. + self._replace(changed) + return + self._apply({"delete": [removed_id]}) + self.refresh() + + def pop(self, index: SupportsIndex = -1) -> Entry: + entry: Final = self[index] + del self[index] + return entry + + def remove(self, value: Entry) -> None: + del self[self.index(value)] + + def clear(self) -> None: + self._replace([]) + + def sort(self, *args: object, **kwargs: object) -> None: # kwargs-ok: list.sort's own arguments + changed: Final = list(self) + changed.sort(*args, **kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # list.sort's own arguments + self._replace(changed) + + def reverse(self) -> None: + self._replace(list(reversed(self))) + + def __imul__(self, count: SupportsIndex) -> "HostedModelList": # pyright: ignore[reportIncompatibleMethodOverride] # list's in-place repeat, as a changeset + self._replace(list(self) * int(count)) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + # A copy or pickle of the live list is a plain list of its entries. + return (list, (list(self),)) diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py new file mode 100644 index 00000000000..a55a1cfc6d6 --- /dev/null +++ b/litellm/router_rust/surface/registry.py @@ -0,0 +1,2191 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportAny=false +# (the ported core code reads untyped litellm helpers: PatternMatchRouter, Deployment.to_json, host.invoke) +""" +``HostRegistry``: the Rust core's live ``model_list`` and the indexes ``registry()`` reads. + +router-core builds the deployments (``litellm-rust/crates/router-core/src/build.rs``): it walks +the list, decides which values are ``os.environ/`` secret references, assigns the id of a +deployment that declares none and adds one deployment per ``organization``. What needs live Python +objects is a host step this registry answers, one deployment at a time (``_answer_build_step``): +``secrets`` (``get_secret``), ``validate`` (the PTU, access-window and strict param checks, and the +pricing each deployment writes into ``litellm.model_cost``), ``provider`` (``get_llm_provider``) +and ``added`` (pass-through credentials, pre-routing strategies, the patterns and indexes below). + +router-core holds the deployments. ``_core_model_list`` is ``HostedModelList`` +(litellm/router_rust/surface/model_list.py): router-core's records read back through the handle, every +write a changeset (a build serves what it added once it ends, a removal is a delete, any other +write a replace), each deployment one Python object while its record is unchanged. + +The indexes are router-core's too (``host_index.rs``), kept by the Python router's history rules: +the id, group and team position maps, ``model_names``, the only-growing ``deployment_names``, +the team public names, the provider default ids and the wildcard pattern routers. The registry +hands them out as the live views of litellm/router_rust/surface/indexes.py, whose writes are changesets. + +The lookups run in router-core (``lookup.rs``): the deployments by name (team public name first), +the model list with its alias, routing group and wildcard rows, the ids, names and access groups +it lists, the candidate ids for a route, the routable deployments, the credential deployment picks +and a team model's alternatives. Each answers with rows, positions in the list: this registry +reads its stored entries at them (``_rows``), so identity and in-place reads stay its own, and a +position router-core could not read (an index a caller wrote junk into) is read here and raises +as the Python router's read raised. What stays here needs live Python objects: the alias map and +routing groups a lookup is scoped by (``_list_scope``), the provider a wildcard request is tried +under, pydantic ``Deployment`` values, the credentials, the PTU ceilings, search tool support, a +team model's provider pick (``provider_for_generic_call``) and the SDK clients. + +Model info and group info are router-core's rules too (``model_info.rs``, ``group_info.rs``), +answered from this process through ``HostModelInfo`` (litellm/router_rust/surface/model_info.py). + +The attribute and method names follow the router core's (``_core_model_list``, +``_core_get_model_list`` ...), so ``litellm.router_api.RouterRegistry`` reads this object as its +router and every ``registry()`` query dispatches the way it does on the Python core. +""" + +import copy +import dataclasses +import os +import re +import threading +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from functools import lru_cache, partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast # noqa: TID251 # entries as a caller wrote them + +from pydantic import TypeAdapter + +from litellm import exceptions as litellm_exceptions +from litellm import get_secret_str +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE +from litellm.litellm_core_utils.access_windows import access_windows_config_error +from litellm.litellm_core_utils.core_helpers import as_str_mapping, without +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_llm_provider_logic import ( + declared_authenticating_provider, + is_registered_custom_provider, + resolve_model_and_provider_for_metadata, +) +from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter +from litellm.litellm_core_utils.ptu_pricing import ( + PTU_COST_ATTRIBUTION_ENV_VAR, + declares_ptu, + declares_ptu_shares, + is_ptu_cost_attribution_enabled, + ptu_config_error, + ptu_identity_error, + ptu_terms, + zeroed_ptu_pricing, +) +from litellm.litellm_core_utils.ptu_shares import ptu_capacity_warning +from litellm.litellm_core_utils.routing_common_utils import ( + provider_for_generic_call, + resolve_model_group_alias, + team_may_use_deployment, + warn_on_ignored_model_capability_flags, + warn_on_provider_credential_mismatch, + with_admin_api_key_secret_reference, + without_model_capability_flags, +) +from litellm.litellm_core_utils.routing_groups import apply_routing_group_priority +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.responses.bridge_check import keep_registered_mode +from litellm.router_api import RouterRegistry +from litellm.router_rust.invoke.shaping import ROUTER_CLIENTS_TARGET +from litellm.router_rust.surface.indexes import HostedIndexes +from litellm.router_rust.surface.model_info import HostModelInfo +from litellm.router_rust.surface.model_list import HostedModelList +from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model +from litellm.router_utils.deployment_numbers import refuse_unservable_numbers, refuse_wide_numbers +from litellm.router_utils.endpoint_eligibility import warn_on_unenforced_endpoint_flag +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import deployment_has_io_token_limits +from litellm.types.router import ( + VALID_LITELLM_ENVIRONMENTS, + CredentialLiteLLMParams, + Deployment, + DeploymentTypedDict, + LiteLLM_Params, + ModelGroupInfo, + RouterModelGroupAliasItem, + RoutingGroup, + RoutingPlugin, + TaggedPreRoutingStrategy, + validate_litellm_params_strictly, +) +from litellm.types.utils import ( + DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS, + CustomPricingLiteLLMParams, + LlmProviders, + LlmProvidersSet, + is_server_derived_pricing_key, + shared_backend_model_info, +) +from litellm.utils import ( + ModelInfoLookupMiss, + catalog_with_runtime_registrations, + get_potential_model_names, + get_secret, + lookup_model_info_for_registration, + model_info_with_pricing_overrides, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.router_python_host import PythonHost + from litellm.router_rust.steps.callbacks import OptionalChecks + from litellm.router_rust.steps.prerouting import PreRoutingStrategies + from litellm.router_strategy.budget_limiter import RouterBudgetLimiting + from litellm.rust_bridge._native import RouterCoreHandle + +# The PTU share teams and the team ceilings read so far. +_PTUMemo: TypeAlias = tuple[ + frozenset[str], + dict[tuple[str, str], "PTUTeamCeiling | None"], +] +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +_CACHE_PRICING_FIELDS: Final = ( + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_200k_tokens", + "cache_read_input_token_cost", + "cache_read_input_token_cost_above_200k_tokens", +) +_OFF_PEAK_BASE_RATE_FIELDS: Final = ( + "input_cost_per_token", + "input_cost_per_second", + "cost_per_second", + "tiered_pricing", +) + +# A stored model_list entry: the live dict the router core keeps. +UntypedEntry: TypeAlias = dict[str, object] +_TeamIndex: TypeAlias = dict[tuple[str, str], list[int]] + +# How a build's deployments join router-core's registry (`build::Install`). +BuildInstall: TypeAlias = Literal["replace", "append", "upsert"] +# Runs one router-core build (`RouterCoreHandle.build`): the entries, the path, whether it resolves +# secrets, how it installs, and the registry's answer to each step. ``RustCore`` supplies it. +BuildRunner: TypeAlias = Callable[ + [Sequence[object], str, bool, BuildInstall, Callable[[Mapping[str, object]], object]], None +] + + +@dataclasses.dataclass(slots=True) +class _BuildEntry: + """One ``set_model_list`` entry as its loop holds it (the router core pops ``model_name``, + ``litellm_params`` and ``model_info`` off the entry and keeps the rest), which router-core's + build steps name by index.""" + + extras: UntypedEntry + model_name: str + litellm_params: UntypedEntry + model_info: UntypedEntry + declared_id: str | None + # `organization` is a list, which router-core adds one deployment for each of. + per_organization: bool = False + # A retried entry keeps whether its id was declared twice in the list it came from. + duplicate: bool | None = None + + +@dataclasses.dataclass(slots=True) +class _Build: + """The state of one router-core build, which ``_answer_build_step`` reads and writes.""" + + path: Literal["create", "add"] + # `create`: the entries (a deep copy, which the build consumes), and each one once router-core + # reaches it. + raw: Sequence[UntypedEntry] = () + # Filled as the build reaches each entry. + entries: dict[int, _BuildEntry] = dataclasses.field(default_factory=dict) # mutable-ok: filled in step order + # `add`: the deployment to add, and the entry it is stored as. + deployment: Deployment | None = None + stored: UntypedEntry | None = None + # The deployment the build is on, its provider and its data-source values. + pending: Deployment | None = None + provider: str | None = None + data_source_values: tuple[tuple[int, str, str], ...] = () + # `add`: the deployment it added. + result: Deployment | None = None + # `create`: the error a deployment raised. The Python core's loop stops there, so every later + # step is answered as invalid and the build ends, then this is raised. + failure: Exception | None = None + + +def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: + """The environment-gating rule (``model_info_is_active_for_environment``): a deployment whose + model_info names ``supported_environments`` loads only where LITELLM_ENVIRONMENT is in it.""" + if model_info is None: + return True + supported_environments: Final = model_info.get("supported_environments") + if supported_environments is None: + return True + if not isinstance(supported_environments, list | tuple): + raise ValueError( # noqa: TRY004 # the core raises ValueError here + f"supported_environments must be a list of {VALID_LITELLM_ENVIRONMENTS}. " + f"but set as: {supported_environments} for model_info: {model_info}" + ) + litellm_environment: Final = get_secret_str(secret_name="LITELLM_ENVIRONMENT") + if litellm_environment is None: + raise ValueError("Set 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .env") + if litellm_environment not in VALID_LITELLM_ENVIRONMENTS: + raise ValueError( + f"LITELLM_ENVIRONMENT must be one of {VALID_LITELLM_ENVIRONMENTS}. but set as: {litellm_environment}" + ) + for environment in supported_environments: # pyright: ignore[reportUnknownVariableType] # a configured list + if environment not in VALID_LITELLM_ENVIRONMENTS: + raise ValueError( + f"supported_environments must be one of {VALID_LITELLM_ENVIRONMENTS}. but set as: {environment} " + f"for model_info: {model_info}" + ) + return litellm_environment in supported_environments + + +# -- pricing written into litellm.model_cost ----------------------------------------------------- + + +def _backend_model_info( + host: "PythonHost", backend_model: str, custom_llm_provider: str | None +) -> Mapping[str, object]: + """``get_model_info`` for the backend model; raises like it for an unmapped model.""" + info: Final = host.invoke("get_model_info")(model=backend_model, custom_llm_provider=custom_llm_provider) + return _MODEL_INFO_ADAPTER.validate_python(info) + + +def _backend_cost_entry( + host: "PythonHost", backend_model: str, custom_llm_provider: str | None +) -> Mapping[str, object] | None: + """The backend model's raw ``litellm.model_cost`` entry, or None when it has none.""" + try: + info: Final = _backend_model_info(host, backend_model, custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model + return None + entry: Final[object] = host.dynamic().model_cost.get(str(info.get("key") or "")) + return _MODEL_INFO_ADAPTER.validate_python(entry) if isinstance(entry, dict) else None + + +def _inherited_cache_pricing( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The cache prices a custom-priced entry inherits (``_inherit_builtin_cache_pricing``).""" + inherited_fields: Final = ( + _CACHE_PRICING_FIELDS + if model_info.get("cache_read_input_token_cost") is not None + else (*_CACHE_PRICING_FIELDS, "explicit_cache_read_input_token_cost") + ) + missing_fields: Final = tuple(field for field in inherited_fields if model_info.get(field) is None) + if not missing_fields: + return {} + try: + backend_info: Final = _backend_model_info(host, backend_model, provider) + except Exception: # noqa: BLE001 # an unmapped backend model contributes nothing + return {} + return {field: backend_info[field] for field in missing_fields if backend_info.get(field) is not None} + + +def _inherited_service_tier_pricing( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The service-tier rates a custom-priced entry inherits (``_inherit_builtin_service_tier_pricing``).""" + if ptu_terms(model_info) is not None and is_ptu_cost_attribution_enabled(): + return {} + if all(model_info.get(field) is None for field in ("input_cost_per_token", "output_cost_per_token")): + return {} + backend_entry: Final = _backend_cost_entry(host, backend_model, provider) + if backend_entry is None: + return {} + return { + field: copy.deepcopy(value) + for field, value in backend_entry.items() + if field.endswith(SERVICE_TIER_COST_KEY_SUFFIXES) and model_info.get(field) is None and value is not None + } + + +def _inherited_off_peak_base_rates( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The whole backend entry an off-peak-only entry copies (``_inherit_builtin_base_rates_for_off_peak``).""" + if not model_info.get("off_peak_pricing"): + return {} + if any(model_info.get(field) is not None for field in _OFF_PEAK_BASE_RATE_FIELDS): + return {} + backend_entry: Final = _backend_cost_entry(host, backend_model, provider) + if backend_entry is None: + return {} + return { + field: copy.deepcopy(value) + for field, value in backend_entry.items() + if model_info.get(field) is None and value is not None + } + + +def _inherited_tiered_output_rate( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The flat output rate a tier table without one inherits (``_inherit_builtin_tiered_output_rate``).""" + tiers: Final = model_info.get("tiered_pricing") + if not isinstance(tiers, list) or not tiers: + return {} + if model_info.get("output_cost_per_token") is not None: + return {} + if all(isinstance(tier, dict) and "output_cost_per_token" in tier for tier in tiers): # pyright: ignore[reportUnknownVariableType] # a configured tier table + return {} + try: + backend_info: Final = _backend_model_info(host, backend_model, provider) + except Exception: # noqa: BLE001 # an unmapped backend model contributes nothing + return {} + backend_rate: Final = backend_info.get("output_cost_per_token") + return {"output_cost_per_token": backend_rate} if backend_rate else {} + + +def _bedrock_profile(params: Mapping[str, object], model: str) -> bool: + return params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/") + + +def _search_tool_support(deployment: Mapping[str, object]) -> bool | None: + """A deployment's answer to Codex's client ``tool_search``.""" + from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request + from litellm.utils import ProviderConfigManager + + configured: Final = _model_info_of(deployment).get("supports_search_tool") + if isinstance(configured, bool): + return configured + params: Final = _params_of(deployment) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return None + try: + resolved_model, resolved_provider = resolve_model_and_provider_for_metadata( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=resolved_model, provider=LlmProviders(resolved_provider) + ) + except (litellm_exceptions.BadRequestError, ValueError): + return None + return True if chat_config_builds_anthropic_request(chat_config) else None + + +def cost_map_backend_model(deployment: Deployment) -> str: + """The model a deployment's prices are inherited from (``_cost_map_backend_model``).""" + model_info_base_model: Final = deployment.model_info.base_model + if isinstance(model_info_base_model, str) and model_info_base_model: + return model_info_base_model + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + params_base_model: Final = params.get("base_model") + if isinstance(params_base_model, str) and params_base_model: + return params_base_model + model: Final = deployment.litellm_params.model + invoked_model: Final = params.get("model_id") + if isinstance(invoked_model, str) and invoked_model and _bedrock_profile(params, model): + profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}" + profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock") + if not isinstance(profile_info, ModelInfoLookupMiss): + return profile_model + return model + + +def deployment_model_cost_payload(host: "PythonHost", deployment: Deployment) -> Mapping[str, object]: + """The ``model_info`` a deployment contributes to ``litellm.model_cost`` + (``_deployment_model_cost_payload``): its model_info, its custom pricing, then the inherited rates.""" + declared: Final = dict(_MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True))) + custom_pricing: Final = { + field: value + for field in CustomPricingLiteLLMParams.model_fields + if (value := deployment.litellm_params.get(field)) is not None + } + backend_model: Final = cost_map_backend_model(deployment) + provider: Final = deployment.model_info.pricing_provider or deployment.litellm_params.custom_llm_provider + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + priced: Final = {**declared, **custom_pricing} + profile_pricing: Final[Mapping[str, object]] = ( + _bedrock_profile_pricing(backend_model, priced) + if params.get("model_id") + and backend_model != deployment.litellm_params.model + and _bedrock_profile(params, deployment.litellm_params.model) + else {} + ) + with_profile: Final = {**priced, **profile_pricing} + off_peak: Final = {**with_profile, **_inherited_off_peak_base_rates(host, with_profile, backend_model, provider)} + cached: Final = ( + {**off_peak, **_inherited_cache_pricing(host, off_peak, backend_model, provider)} + if off_peak.get("input_cost_per_token") is not None + else off_peak + ) + tiers: Final = {**cached, **_inherited_service_tier_pricing(host, cached, backend_model, provider)} + return {**tiers, **_inherited_tiered_output_rate(host, tiers, backend_model, provider)} + + +def _bedrock_profile_pricing(backend_model: str, priced: Mapping[str, object]) -> Mapping[str, object]: + profile_info: Final = lookup_model_info_for_registration(model=backend_model, custom_llm_provider="bedrock") + if isinstance(profile_info, ModelInfoLookupMiss): + return {} + explicit_info: Final = _MODEL_INFO_ADAPTER.validate_python(priced) + priced_info: Final = model_info_with_pricing_overrides(profile_info, explicit_info) # pyright: ignore[reportArgumentType] # the explicit model_info, as the core passes it + return {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} + + +def _backend_cost_map_keys(model: str, custom_llm_provider: str | None) -> tuple[str, ...]: + """The ``litellm.model_cost`` keys a deployment's shared backend info is registered under.""" + backend_key: Final = model if custom_llm_provider is None else f"{custom_llm_provider}/{model}" + if "responses/" in backend_key: + return (backend_key, backend_key.replace("responses/", "")) + return (backend_key,) + + +def _catalog_entry_has_token_limit(catalog_entry: Mapping[str, object], field: str) -> bool: + return catalog_entry.get(field) is not None or ( + field == "max_tokens" and catalog_entry.get("max_output_tokens") is not None + ) + + +def _shared_backend_model_info_for_keys( + catalog: Mapping[str, Mapping[str, object]], + backend_keys: tuple[str, ...], + shared_model_info: Mapping[str, object], +) -> Mapping[str, object]: + """``shared_model_info`` without a deployment's token limit the catalog already sets on one of + ``backend_keys`` (``_shared_backend_model_info_for_keys``).""" + catalog_token_limit_fields: Final = frozenset( + field + for field in DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS + if any(_catalog_entry_has_token_limit(catalog.get(key) or {}, field) for key in backend_keys) + ) + return {field: value for field, value in shared_model_info.items() if field not in catalog_token_limit_fields} + + +def _catalog_backend_keys( + model: str, custom_llm_provider: str | None, backend_keys: tuple[str, ...] +) -> tuple[str, ...]: + """Every name the catalog may list the backend model under. The provider is the declared one: + resolving it would reach provider auth (``catalog_backend_keys``).""" + model_provider_prefix: Final = model.partition("/")[0] + catalog_provider: Final = ( + custom_llm_provider + if custom_llm_provider is not None + else model_provider_prefix + if model_provider_prefix in LlmProvidersSet + else None + ) + if catalog_provider is None: + return (*backend_keys, model) + potential_names: Final = get_potential_model_names(model, catalog_provider) + return ( + *backend_keys, + model, + potential_names["split_model"], + potential_names["stripped_model_name"], + potential_names["combined_model_name"], + potential_names["combined_stripped_model_name"], + potential_names["provider_prefixed_model_name"], + potential_names["region_free_combined_model_name"], + ) + + +def register_deployment_in_model_cost( + host: "PythonHost", + *, + model_id: str | None, + model_info: Mapping[str, object], + model: str, + custom_llm_provider: str | None, + pricing_provider: str | None = None, +) -> None: + """Write a deployment's metadata into ``litellm.model_cost`` (``_register_deployment_in_model_cost``): + the whole entry under its id, the schema fields under its backend model key(s).""" + registered: Final = ( + without(model_info, *CustomPricingLiteLLMParams.model_fields) + if classify_strategy_router_model(model) is not None + else dict(model_info) + ) + register_model: Final = host.invoke("register_model") + if model_id is not None: + register_model( + model_cost={model_id: registered}, + persist_across_reloads=False, + warning_display_name=model, + custom_llm_provider=pricing_provider or custom_llm_provider, + add_to_provider_model_sets=False, + ) + backend_keys: Final = _backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) + shared_model_info: Final = _shared_backend_model_info_for_keys( + catalog=catalog_with_runtime_registrations(), + backend_keys=_catalog_backend_keys(model, custom_llm_provider, backend_keys), + shared_model_info=shared_backend_model_info(registered), + ) + register_model( + model_cost={key: keep_registered_mode(key, shared_model_info) for key in backend_keys}, + persist_across_reloads=False, + ) + + +def register_deployment_pricing(host: "PythonHost", deployment: Deployment) -> None: + """Register a deployment's custom and inherited pricing (``_register_deployment_pricing``).""" + register_deployment_in_model_cost( + host, + model_id=deployment.model_info.id, + model_info=deployment_model_cost_payload(host, deployment), + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + pricing_provider=deployment.model_info.pricing_provider, + ) + + +# -- the registry -------------------------------------------------------------------------------- + + +def _model_info_of(entry: Mapping[str, object]) -> Mapping[str, object]: + model_info: Final[object] = entry.get("model_info") + return _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else {} + + +def _params_of(entry: Mapping[str, object]) -> Mapping[str, object]: + params: Final[object] = entry.get("litellm_params") + return _LITELLM_PARAMS_ADAPTER.validate_python(params) if isinstance(params, dict) else {} + + +def _renamed(entry: UntypedEntry, model_name: str) -> UntypedEntry: + return {**entry, "model_name": model_name} + + +def _as_routing_group_row(deployment: Mapping[str, object]) -> UntypedEntry: + """A member deployment under its group's name, without the member's ``access_groups``.""" + model_info: Final = without(_model_info_of(deployment), "access_groups") + return {**deployment, "model_info": model_info} + + +def _ref_env_only(ref: object) -> bool: + return bool((as_str_mapping(ref) or {}).get("env_only")) + + +def _duplicate_of(step: Mapping[str, object], entry: _BuildEntry) -> bool: + """Whether the entry's declared id is declared twice in its list: a retried entry keeps the + answer its first build had.""" + return entry.duplicate if entry.duplicate is not None else step.get("duplicate_id") is True + + +def _index_key(value: object) -> str | None: + """``value`` as a key of the router's name and id indexes: a dict lookup hashes any key and only + a string can match, so another hashable key finds nothing (``None``), and an unhashable one + raises as the lookup raises. The native lookups take strings alone.""" + _ = hash(value) + return value if isinstance(value, str) else None + + +class HostRegistry(HostModelInfo): + """The live model_list and its indexes, kept the way the Python router core keeps them.""" + + def __init__( + self, + host: "PythonHost", + *, + model_group_alias: "dict[str, str | RouterModelGroupAliasItem]", # mutable-ok: the router's live alias map + ignore_invalid_deployments: bool, + on_change: Callable[[], None], + ) -> None: + self._host: Final = host + self._on_change: Final = on_change + self.ignore_invalid_deployments: bool = ignore_invalid_deployments + self._registry_version: int = 0 + self._deployment_index_lock: Final = threading.RLock() + self._core_model_group_alias: dict[str, str | RouterModelGroupAliasItem] = model_group_alias # mutable-ok: the live alias map # fmt: skip + # The live model_list: router-core's deployments read back, every write a changeset. + self._core_model_list: Final[HostedModelList] = HostedModelList(self._live_handle, self._hand_changes) + # The indexes the registry hands out: router-core's, read back, every write a changeset. + self._indexes: Final = HostedIndexes(self._live_handle, self._hand_changes) + self._core_deployment_latency_map: dict[str, float] = {} # mutable-ok: the live latency map + self._access_groups_cache: Mapping[str, Sequence[str]] | None = None + # The routing groups by name; RustCore sets them from its resolved settings. + self._core_routing_groups: dict[str, RoutingGroup] = {} # mutable-ok: the routing_groups attribute + self._routing_group_rows: tuple[UntypedEntry, ...] | None = None + self._provider_unresolved_deployments: tuple[Callable[[], None], ...] = () + # Runs router-core's deployment build; RustCore sets it before the first build. + self.build_runner: BuildRunner | None = None + self.missing_attribute: Callable[[str], AttributeError] | None = None + # Hands router-core a write its build did not make (a removal, a swapped list) as a + # changeset, and reads its current handle; RustCore sets both before the first build. + self.changes_runner: Callable[[Mapping[str, object]], None] | None = None + self.handle_lookup: Callable[[], RouterCoreHandle | None] | None = None + # The pre-routing strategy registries (``registry().plugins``) and the hooks that keep them; + # RustCore installs its ``PreRoutingStrategies`` (litellm/router_rust/steps/prerouting.py). + self._core_auto_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: the live registry + self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self.strategies: PreRoutingStrategies | None = None + # The router's optional pre-call checks (deployment budgets, prompt caching pins); + # RustCore installs its ``OptionalChecks`` (litellm/router_rust/steps/callbacks.py). + self.checks: OptionalChecks | None = None + # The Rust core's resolution for a request (`_core_get_routable_deployments`), set by the core. + self.routable_lookup: ( + Callable[[str, Mapping[str, object], str | None], Sequence[Mapping[str, object]]] | None + ) = None + # A deployment's cached `max_parallel_requests` limit (litellm/router_rust/invoke/slots.py), set by the core. + self.max_parallel_requests_lookup: Callable[[Mapping[str, object], Mapping[str, object]], object] | None = None + # `plugins("routing")`: the router's routing plugins, set by the core. + self._core_routing_plugins: list[RoutingPlugin] = [] # mutable-ok: the Python core hands out its live list + # `zero_cost()`: the per-deployment zero-cost lookup the auth checks fill, emptied with the + # group info caches. + self._zero_cost_cache: dict[str, bool] = {} # mutable-ok: the auth checks write into it + # The routing cache usage reads come from; RustCore sets it to the one its Rust core writes. + self._core_cache: DualCache = DualCache() + self._discovered_model_info_cache: InMemoryCache = InMemoryCache( + max_size_in_memory=1, default_ttl=2 * MODEL_INFO_REFRESH_SECONDS + ) + # The PTU share teams and team ceilings, until the model list next changes. + self._ptu_cache: _PTUMemo | None = None + self._cached_get_model_group_info: Callable[[str], ModelGroupInfo | None] = lru_cache( # pyright: ignore[reportIncompatibleMethodOverride] # the per-router cache shadows the uncached method, as the core keeps it + maxsize=DEFAULT_MAX_LRU_CACHE_SIZE + )(self._core_get_model_group_info) + self._core_cached_deployment_model_info: Callable[[str, str], object] = lru_cache( + maxsize=DEFAULT_MAX_LRU_CACHE_SIZE + )(self._core_get_deployment_model_info) + self.cached_deployment_tracks_usage: Callable[[str, str], bool] = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._deployment_tracks_usage + ) + + def __getattr__(self, name: str) -> object: + from litellm.router_rust.core import RustCoreUnsupported + + missing: Final = self.__dict__.get("missing_attribute") + if name == "_core_model_group_alias" and missing is not None: + # `del router.model_group_alias`: every read raises, as on the Python core. + raise missing(name) + raise RustCoreUnsupported(f"registry().{name.removeprefix('_core_')} is not served by the Rust router core") + + # -- building --------------------------------------------------------------------------------- + + def build(self, model_list: Sequence[object] | None) -> None: + """The constructor's registry: ``set_model_list`` and the latency map, as the core builds them.""" + if model_list is None: + return + self._discovered_model_info_cache = InMemoryCache( + max_size_in_memory=max(len(model_list), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS + ) + self.set_model_list(model_list) + for entry in model_list: + params = _params_of(_LITELLM_PARAMS_ADAPTER.validate_python(entry)) + if "model" in params: + self._core_deployment_latency_map[str(params["model"])] = 0 + + async def arefresh_model_info(self, client: "AsyncHTTPHandler | None") -> None: + """``_core_arefresh_model_info``: token limits the served OpenAI-compatible deployments + advertise on their models endpoint, read with a few workers over a snapshot of the list.""" + import asyncio + + from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY + + deployments: Final = iter(tuple(self._core_model_list)) + + async def refresh_worker() -> None: + for raw_deployment in deployments: + try: + await self._arefresh_deployment_model_info(raw_deployment, client=client) + except Exception: # noqa: BLE001 # one invalid deployment must not prevent refreshing the others + verbose_router_logger.debug("Could not refresh deployment model info") + + _ = await asyncio.gather(*(refresh_worker() for _ in range(MODEL_INFO_REFRESH_CONCURRENCY))) + self._invalidate_model_group_info_cache() + + async def _arefresh_deployment_model_info( + self, raw_deployment: Mapping[str, object], *, client: "AsyncHTTPHandler | None" + ) -> None: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.llms.openai_like.model_info import ( + MODEL_INFO_DISCOVERY_PROVIDERS, + get_openai_compatible_model_info, + ) + from litellm.types.router import DiscoveredDeploymentModelInfo + from litellm.utils import aresolve_api_key_secret_reference + + deployment: Final = Deployment.model_validate(raw_deployment) + params: Final = LiteLLM_Params.model_validate( + MappingProxyType( + { + **deployment.litellm_params.model_dump(exclude_none=True), + **( + self._core_get_deployment_credentials_with_provider(deployment.model_info.id or "") + or MappingProxyType({}) + ), + } + ) + ) + model, provider, dynamic_api_key, api_base = self._host.invoke("get_llm_provider")( # pyright: ignore[reportAny] # litellm's untyped tuple + model=params.model, litellm_params=params + ) + if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return + if api_base is None or "*" in model or params.get("use_clientside_credentials"): # pyright: ignore[reportUnknownMemberType] # the params' untyped get + return + api_key: Final = await aresolve_api_key_secret_reference( + params.api_key or dynamic_api_key, # pyright: ignore[reportAny] # see above + model=model, # pyright: ignore[reportAny] # see above + llm_provider=provider, # pyright: ignore[reportAny] # see above + upstream=MappingProxyType({"api_base": params.api_base}), + ) + headers: Final = TypeAdapter(Mapping[str, str]).validate_python( + params.get("extra_headers") or params.get("headers") or MappingProxyType({}) # pyright: ignore[reportUnknownMemberType] # see above + ) + auth_headers: Final = ( + MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) + ) + limits: Final = await get_openai_compatible_model_info( + model=model, # pyright: ignore[reportAny] # see above + api_base=api_base, # pyright: ignore[reportAny] # see above + headers=MappingProxyType( + {**auth_headers, **MappingProxyType({key.lower(): value for key, value in headers.items()})} + ), + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self._core_cache.in_memory_cache, + ) + model_id: Final = deployment.model_info.id + if not limits or model_id is None or self._core_get_model_info(model_id) is not raw_deployment: + return + self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1) + self._discovered_model_info_cache.delete_cache(model_id) + self._discovered_model_info_cache.set_cache( + model_id, + DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits), # pyright: ignore[reportArgumentType] # the model_list entry, as the core keeps it + ) + self._invalidate_model_group_info_cache() + + def _deployment_tracks_usage(self, deployment_id: str, model_group: str) -> bool: + """``_deployment_tracks_usage``: a limit on the deployment or in its merged model info, where + the cost map's own ``tpm`` and ``rpm`` count too.""" + deployment_info: Final = self._core_get_deployment(model_id=deployment_id) + if deployment_info is None: + return False + deployment_model_info: Final = self._core_get_router_model_info( + deployment=deployment_info, + received_model_name=model_group, + ) + configured_limits: Final[tuple[object, ...]] = ( + deployment_info.get("tpm", None), + deployment_info.get("rpm", None), + deployment_info.litellm_params.tpm, + deployment_info.litellm_params.rpm, + deployment_model_info.get("tpm", None), + deployment_model_info.get("rpm", None), + ) + return any(limit is not None for limit in configured_limits) or deployment_has_io_token_limits( + deployment_info.model_dump() + ) + + def _invalidate_model_group_info_cache(self) -> None: + self._ptu_cache = None + self._cached_get_model_group_info.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache + self._core_cached_deployment_model_info.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache + self.cached_deployment_tracks_usage.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache + self._zero_cost_cache.clear() + self._routing_group_rows = None + + def set_routing_groups(self, groups: Sequence[RoutingGroup]) -> None: + """``_init_routing_groups``' registry side: the groups by name, and the caches they feed.""" + self._core_routing_groups = {group.group_name: group for group in groups} + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + + def _invalidate_access_groups_cache(self) -> None: + self._access_groups_cache = None + self._registry_version += 1 + + def set_model_list(self, model_list: Sequence[object]) -> None: + """Replace every deployment (``set_model_list``). All or nothing: when an entry raises, + the deployments served before the call are put back and the error is raised.""" + previous: Final = [copy.deepcopy(dict(entry)) for entry in self._core_model_list] + try: + self._replace_model_list(model_list) + except Exception: + self._replace_model_list(previous) + raise + + def _replace_model_list(self, model_list: Sequence[object]) -> None: + self._discovered_model_info_cache.flush_cache() + original_model_list: Final[list[UntypedEntry]] = copy.deepcopy(list(model_list)) # pyright: ignore[reportAssignmentType] # model_list entries # mutable-ok: a deep copy the build consumes + if self.strategies is not None: + self.strategies.reset() + self._provider_unresolved_deployments = () + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + ptu_declared: Final = tuple( + str(entry.get("model_name")) + for entry in original_model_list + if (ptu_info := _model_info_of(entry)).get("db_model") is not True and declares_ptu(ptu_info) + ) + if ptu_declared and not is_ptu_cost_attribution_enabled(): + verbose_router_logger.warning( + "PTU fields are set on config.yaml deployment(s) %s, but PTU cost attribution is disabled, so no " + "flat cost accrues and this traffic is billed per token. Set %s=True to enable it", + ", ".join(ptu_declared), + PTU_COST_ATTRIBUTION_ENV_VAR, + ) + # router-core walks the list (one deployment per organization, secrets resolved first, + # ids assigned), asking this registry about each deployment in turn; it starts the list + # and its id, name and team indexes over, and takes the given names as `model_names`. + self._run_build( + _Build(path="create", raw=original_model_list), + original_model_list, + resolve_secrets=True, + install="replace", + ) + if self.strategies is not None: + self.strategies.finalize() + self._on_change() + + # -- the build's steps (router-core's `build::build`) ------------------------------------------ + + def _run_build( + self, build: _Build, entries: Sequence[object], *, resolve_secrets: bool, install: BuildInstall + ) -> None: + """Have router-core build ``entries``, answering its steps from this registry.""" + if self.build_runner is None: + raise RuntimeError("the host registry builds deployments through its Rust core") + for entry in entries: + refuse_wide_numbers(entry) + self._core_model_list.begin_build(replacing=install == "replace") + try: + self.build_runner(entries, build.path, resolve_secrets, install, partial(self._answer_shown, build)) + finally: + self._core_model_list.end_build() + if build.failure is not None: + # router-core served what it built before the entry that raised (`set_model_list` + # cleared the list first), so the router is left as the Python core leaves it. + raise build.failure + + def _answer_shown(self, build: _Build, step: Mapping[str, object]) -> object: + """A build step's answer; a deployment the ``added`` step hands router-core shows in the + live list from then on (router-core serves it when the build ends).""" + answer: Final = self._answer_build_step(build, step) + stored: Final = as_str_mapping(answer).get("entry") if step["kind"] == "added" else None # pyright: ignore[reportOptionalMemberAccess] # an added step answers a mapping + if isinstance(stored, Mapping): + self._core_model_list.built(cast(Mapping[str, object], stored)) # cast-ok: the entry it stores + return answer + + def _answer_build_step(self, build: _Build, step: Mapping[str, object]) -> object: + """The answer to one of router-core's build steps. A ``set_model_list`` deployment that + fails is left out when the router ignores invalid deployments (``_create_deployment``), + and retried on the next price data reload when its provider did not resolve.""" + if build.failure is not None: + return {"invalid": True} + kind: Final = step["kind"] + index: Final = int(str(step["index"])) + refs: Final = step.get("refs") + if kind == "secrets" and isinstance(refs, list) and not any(_ref_env_only(ref) for ref in refs): + return self._build_secrets(build, index, refs) + if kind == "validate": + # A new deployment: the entry is popped where the loop pops it, outside the error + # handling, and nothing of the last deployment carries over. + build.pending = None # rebind-ok: the build's state, which each step advances + build.provider = None # rebind-ok: the build's state, which each step advances + build.data_source_values = () # rebind-ok: the build's state, which each step advances + if build.path == "create": + _ = self._build_entry(build, index) + try: + if kind == "secrets": + return self._build_data_sources(build, refs if isinstance(refs, list) else []) + if kind == "validate": + return self._build_validate(build, index, step) + if kind == "provider": + build.provider = self._build_provider(build) # rebind-ok: the build's state, which each step advances + return {"provider": build.provider} + return self._build_added(build) + except Exception as e: + if build.path != "create": + raise + if not self.ignore_invalid_deployments: + build.failure = e # rebind-ok: raised once router-core ends the build + return {"invalid": True} + if isinstance(e, litellm_exceptions.BadRequestError): + entry: Final = build.entries[index] + retried: Final = dataclasses.replace(entry, duplicate=_duplicate_of(step, entry)) + self._provider_unresolved_deployments = ( + *self._provider_unresolved_deployments, + partial(self._retry_create, retried), + ) + verbose_router_logger.exception( + "Error creating deployment: %s, ignoring and continuing with other deployments.", e + ) + return {"invalid": True} + + def _build_entry(self, build: _Build, index: int) -> _BuildEntry: + """Entry ``index`` as ``set_model_list``'s loop holds it, popped off the first time a step + names it (so a malformed entry raises where the loop raises).""" + held: Final = build.entries.get(index) + if held is not None: + return held + model: Final = build.raw[index] + model_name: Final = model.pop("model_name") + raw_params: Final = model.pop("litellm_params") + litellm_params: Final[UntypedEntry] = raw_params if isinstance(raw_params, dict) else {} # pyright: ignore[reportAssignmentType] # configured params + model_info: Final = model.pop("model_info", {}) + info_dict: Final[UntypedEntry] = model_info if isinstance(model_info, dict) else {} # pyright: ignore[reportAssignmentType] # a configured model_info + entry: Final = _BuildEntry( + extras=model, + model_name=str(model_name), + litellm_params=litellm_params, + model_info=info_dict, + declared_id=None if info_dict.get("id") is None else str(info_dict["id"]), + per_organization=isinstance(litellm_params.get("organization"), list), + ) + build.entries[index] = entry # rebind-ok: the build's state, which each step advances + return entry + + def _build_secrets(self, build: _Build, index: int, refs: Sequence[object]) -> Sequence[object]: + """``set_model_list``'s ``get_secret`` for each top-level ``os.environ/`` value router-core + found, written into the entry's params.""" + entry: Final = self._build_entry(build, index) + references: Final = tuple(as_str_mapping(ref) or {} for ref in refs) + values: Final = tuple(get_secret(str(reference["name"])) for reference in references) + entry.litellm_params.update( + {str(reference["key"]): value for reference, value in zip(references, values, strict=True)} + ) + return values + + def _build_data_sources(self, build: _Build, refs: Sequence[object]) -> Sequence[object]: + """``_add_deployment``'s environment reads for ``dataSources[].parameters``, applied to the + deployment where it applied them (``_build_added``).""" + references: Final = tuple(as_str_mapping(ref) or {} for ref in refs) + build.data_source_values = tuple( # rebind-ok: the build's state, which each step advances + ( + int(str(reference["source"])), + str(reference["key"]), + os.environ.get(str(reference["name"]).replace("os.environ/", ""), ""), + ) + for reference in references + ) + return tuple(value for _, _, value in build.data_source_values) + + def _build_validate(self, build: _Build, index: int, step: Mapping[str, object]) -> Mapping[str, object]: + if build.path == "add": + return self._validate_added_deployment(build) + entry: Final = self._build_entry(build, index) + routed: Final = as_str_mapping(step["entry"]) or {} + routed_info: Final = as_str_mapping(routed.get("model_info")) or {} + if "id" not in entry.model_info: + entry.model_info["id"] = routed_info.get("id") + if entry.per_organization: + entry.litellm_params["organization"] = (as_str_mapping(routed.get("litellm_params")) or {}).get( + "organization" + ) + duplicate: Final = _duplicate_of(step, entry) + deployment: Final = self._validated_deployment(entry, taken=duplicate) + if deployment is None: + return {"skip": True} + served: Final = with_admin_api_key_secret_reference(deployment) + build.pending = served # rebind-ok: the build's state, which each step advances + # A dump of this core's own: a `mock_response` ModelResponse warns, and only Python's dumps may. + return {"entry": build.pending.to_json(exclude_none=True, warnings=False)} + + def _validated_deployment(self, entry: _BuildEntry, *, taken: bool) -> Deployment | None: + """``_create_deployment`` up to ``_add_deployment``: the config checks, the deployment, its + pricing, and None when the environment gate leaves it out.""" + model_name: Final = entry.model_name + litellm_params: Final = entry.litellm_params + model_info: Final = entry.model_info + config_sourced: Final = model_info.get("db_model") is not True + identity_error: Final = ( + ptu_identity_error( + declared_id=entry.declared_id, + taken=taken, + current_id=model_info.get("id"), # pyright: ignore[reportArgumentType] # the configured id + model_name=model_name, + ) + if config_sourced and ptu_terms(model_info) is not None + else None + ) + ptu_error: Final = ( + (ptu_config_error(model_info, model_name=model_name) or identity_error) if config_sourced else None + ) + if ptu_error is not None and (declares_ptu_shares(model_info) or is_ptu_cost_attribution_enabled()): + raise ValueError(ptu_error) + access_windows_error: Final = access_windows_config_error(model_info, model_name=model_name) + if access_windows_error is not None: + raise ValueError(access_windows_error) + capacity_warning: Final = ( + ptu_capacity_warning( + model_name, MappingProxyType({"model_info": model_info, "litellm_params": litellm_params}) + ) + if is_ptu_cost_attribution_enabled() + else None + ) + if capacity_warning is not None: + verbose_router_logger.warning(capacity_warning) + zeroed_pricing: Final = zeroed_ptu_pricing(model_info, litellm_params) if config_sourced else None + merged_params: Final[Mapping[str, object]] = ( + litellm_params if zeroed_pricing is None else MappingProxyType({**litellm_params, **zeroed_pricing}) + ) + params: Final[LiteLLM_Params] = validate_litellm_params_strictly(LiteLLM_Params, dict(**merged_params)) + declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(litellm_params) + warn_on_provider_credential_mismatch(model_name=model_name, litellm_params=declared_params) + warn_on_ignored_model_capability_flags(model_name=model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag(model_name=model_name, model_info=model_info) + deployment: Final = without_model_capability_flags( + Deployment(**entry.extras, model_name=model_name, litellm_params=params, model_info=model_info) # pyright: ignore[reportArgumentType] # the entry's extra fields, as the core passes them + ) + # The values as typed: a cost written as "inf", a share written as 1e20. + refuse_unservable_numbers(deployment.to_json(exclude_none=True, warnings=False)) + register_deployment_pricing(self._host, deployment) + if model_info_is_active_for_environment(model_info=deployment.model_info) is not True: # pyright: ignore[reportArgumentType] # ModelInfo reads like a mapping + verbose_router_logger.warning( + "Ignoring deployment %s as it is not active for environment %s", + deployment.model_name, + deployment.model_info["supported_environments"], + ) + return None + for pattern in deployment.litellm_params.get("tag_regex") or []: + try: + re.compile(pattern) + except re.error as exc: + raise ValueError( + f"Invalid regex in tag_regex for model '{deployment.model_name}': {pattern!r} — {exc}" + ) from exc + return deployment + + def _validate_added_deployment(self, build: _Build) -> Mapping[str, object]: + """``add_deployment`` up to ``_add_deployment``: an id the registry holds is skipped.""" + deployment: Final = build.deployment + if deployment is None: + raise RuntimeError("an add build holds its deployment") + if deployment.model_info.id and self._core_has_model_id(deployment.model_info.id): + return {"skip": True} + declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) + warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) + warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag( + model_name=deployment.model_name, model_info=deployment.model_info.model_dump() + ) + served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) + build.pending = served # rebind-ok: the build's state, which each step advances + build.stored = served.to_json(exclude_none=True) # rebind-ok: the build's state, which each step advances + refuse_unservable_numbers(build.stored) + return {"entry": build.stored} + + def _build_provider(self, build: _Build) -> str | None: + """``_add_deployment``'s provider: None for a prompt-management model, else the provider + ``get_llm_provider`` resolves, refused when litellm does not serve it.""" + deployment: Final = build.pending + if deployment is None: + raise RuntimeError("a provider step follows a validated deployment") + litellm_model: Final = deployment.litellm_params.model + if isinstance(deployment.litellm_params.drop_params, str): + verbose_router_logger.warning( + "model=%s drop_params=%r is not a flag value, treating it as unset", + deployment.model_name, + deployment.litellm_params.drop_params, + ) + dynamic: Final = self._host.dynamic() + if "/" in litellm_model and litellm_model.split("/")[0] in dynamic.known_custom_logger_compatible_callbacks: + return None + if declared_authenticating_provider(litellm_model, deployment.litellm_params.custom_llm_provider) is not None: + _, provider = self._model_group_llm_provider(deployment.litellm_params) + return provider + _, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.get("custom_llm_provider", None), + api_base=deployment.litellm_params.api_base, + litellm_params=deployment.litellm_params, + ) + if ( + custom_llm_provider not in dynamic.provider_list + and not JSONProviderRegistry.exists(custom_llm_provider) + and not is_registered_custom_provider(custom_llm_provider) + ): + raise Exception(f"Unsupported provider - {custom_llm_provider}") # noqa: TRY002 # the core raises a plain Exception + return custom_llm_provider + + def _build_added(self, build: _Build) -> Mapping[str, object]: + """The rest of ``_add_deployment``: the entry router-core stores, which joins the list and + the indexes (``host_index.rs``) when the step returns.""" + deployment: Final = build.pending + if deployment is None: + raise RuntimeError("an added step follows a validated deployment") + self._register_deployment(deployment, build.provider, build.data_source_values) + build.pending = None # rebind-ok: the build's state, which each step advances + build.data_source_values = () # rebind-ok: the build's state, which each step advances + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + if build.path == "create": + return {"entry": deployment.to_json(exclude_none=True)} + entry: Final = build.stored if build.stored is not None else deployment.to_json(exclude_none=True) + register_deployment_pricing(self._host, deployment) + build.result = deployment # rebind-ok: the build's state, which each step advances + return {"entry": entry} + + def _retry_create(self, entry: _BuildEntry) -> None: + """A ``set_model_list`` deployment left out when its provider did not resolve, built again + and added at the end.""" + build: Final = _Build(path="create") + build.entries[0] = entry + self._run_build( + build, + [ + { + **entry.extras, + "model_name": entry.model_name, + "litellm_params": entry.litellm_params, + "model_info": entry.model_info, + } + ], + resolve_secrets=False, + install="append", + ) + + @staticmethod + def _model_group_llm_provider(litellm_params: LiteLLM_Params) -> tuple[str, str]: + try: + return resolve_model_and_provider_for_metadata( + model=litellm_params.model, custom_llm_provider=litellm_params.custom_llm_provider + ) + except litellm_exceptions.BadRequestError as e: + verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) + return "", "" + + def _register_deployment( + self, + deployment: Deployment, + custom_llm_provider: str | None, + data_source_values: Sequence[tuple[int, str, str]], + ) -> None: + """``_add_deployment`` after the provider: its data-source values, pass-through + credentials and pre-routing strategies (router-core registers its names and patterns).""" + if deployment.litellm_params.rpm is None and getattr(deployment, "rpm", None) is not None: + deployment.litellm_params.rpm = getattr(deployment, "rpm") # noqa: B009 # an extra field of the entry # rebind-ok: the core sets it on the deployment it adds + if deployment.litellm_params.tpm is None and getattr(deployment, "tpm", None) is not None: + deployment.litellm_params.tpm = getattr(deployment, "tpm") # noqa: B009 # an extra field of the entry # rebind-ok: see above + data_sources: Final = deployment.litellm_params.get("dataSources", []) or [] + for source, key, value in data_source_values: + data_sources[source]["parameters"][key] = value + if custom_llm_provider is not None: + self._initialize_deployment_for_pass_through(deployment, custom_llm_provider) + if self.strategies is not None: + self.strategies.register(deployment) + + def _initialize_deployment_for_pass_through(self, deployment: Deployment, custom_llm_provider: str) -> None: + """Register vertex credentials for a pass-through deployment (``_initialize_deployment_for_pass_through``).""" + if deployment.litellm_params.use_in_pass_through is not True or custom_llm_provider != "vertex_ai": + return + credential_name: Final = deployment.litellm_params.litellm_credential_name + credential_values: Final = ( + CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {} + ) + project: Final = credential_values.get("vertex_project") or deployment.litellm_params.vertex_project + location: Final = credential_values.get("vertex_location") or deployment.litellm_params.vertex_location + credentials: Final = credential_values.get("vertex_credentials") or deployment.litellm_params.vertex_credentials + if project is None or location is None: + raise ValueError( + "vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints." + ) + from litellm.router_host import router_host + + router_host().add_vertex_pass_through_credentials( + project_id=str(project), + location=str(location), + vertex_credentials=credentials, # pyright: ignore[reportArgumentType] # the configured credentials + ) + + # -- writes ----------------------------------------------------------------------------------- + + def add_deployment(self, deployment: Deployment) -> Deployment | None: + """Add one deployment unless its id exists (``add_deployment``), built by router-core.""" + build: Final = _Build(path="add", deployment=deployment) + self._run_build(build, [deployment.to_json(exclude_none=True)], resolve_secrets=False, install="upsert") + if build.result is None: + return None + if self.checks is not None: + self.checks.sync_deployment_budget(build.result) + self._on_change() + return build.result + + def _core_upsert_deployment(self, deployment: Deployment) -> Deployment | None: + """Add or replace one deployment (``_upsert_deployment``).""" + model_id: Final = deployment.model_info.id or "" + on_router: Final = self._core_get_deployment(model_id=model_id) + served: Final = without_model_capability_flags(deployment) + try: + if on_router is not None: + if ( + deployment.model_name == on_router.model_name + and served.litellm_params == on_router.litellm_params + and deployment.model_info == on_router.model_info + ): + return None + with self._deployment_index_lock: + removal_idx: Final = self.model_id_to_deployment_index_map.get(model_id) + if removal_idx is not None: + _ = self._core_model_list[removal_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does + self._remove(model_id) + if self.strategies is not None: + self.strategies.release(on_router) + self._on_change() + self.add_deployment(deployment=deployment) + if self.strategies is not None: + self.strategies.upserted(deployment, on_router) + except Exception as e: + if not self.ignore_invalid_deployments: + raise + verbose_router_logger.warning( + "Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.", + deployment.model_name, + deployment.model_info.id, + e, + ) + self._restore_deployment_after_failed_upsert(on_router, model_id) + return None + else: + return deployment + + def _restore_deployment_after_failed_upsert(self, previous: Deployment | None, model_id: str) -> None: + if previous is None or self._core_has_model_id(model_id): + return + try: + if self.strategies is None: + self.add_deployment(deployment=previous) + else: + self.strategies.without_capability_limit(lambda: self.add_deployment(deployment=previous)) + verbose_router_logger.info( + "Restored deployment %s (id=%s); it keeps serving its previous configuration.", + previous.model_name, + model_id, + ) + except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload + verbose_router_logger.warning( + "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s", + previous.model_name, + model_id, + restore_error, + ) + + def _live_handle(self) -> "RouterCoreHandle | None": + return None if self.handle_lookup is None else self.handle_lookup() + + def _hand_changes(self, changes: Mapping[str, object]) -> None: + if self.changes_runner is None: + raise RuntimeError("the host registry writes deployments through its Rust core") + self.changes_runner(changes) + self._core_model_list.refresh() + + def refresh(self) -> None: + """Read router-core's list again after a write (only what was appended, when that is all + that changed). The index views are read again when an entry point hands them out, which + keeps one ``add_deployment`` from reading every index back.""" + self._core_model_list.refresh() + + def _remove(self, model_id: str) -> None: + """Remove the deployment at the id map's position for ``model_id``, every index shifted + (``_update_deployment_indices_after_removal``, in router-core).""" + self._hand_changes({"remove": model_id}) + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + + # -- the indexes (router-core's, read back as live views) ------------------------------------- + + @property + def model_id_to_deployment_index_map(self) -> dict[str, int]: # mutable-ok: the live id index + self._indexes.refresh() + return cast(dict[str, int], self._indexes.id_map) # cast-ok: id to position, as router-core keeps it + + @property + def _core_model_name_to_deployment_indices(self) -> dict[str, list[int]]: # mutable-ok: the live name index + self._indexes.refresh() + return cast(dict[str, list[int]], self._indexes.model_name) # cast-ok: name to positions + + @_core_model_name_to_deployment_indices.setter + def _core_model_name_to_deployment_indices(self, value: object) -> None: + self._indexes.assign("model_name", value) + + @property + def _core_team_model_to_deployment_indices( + self, + ) -> dict[tuple[str, str], list[int]]: # mutable-ok: the live team index + self._indexes.refresh() + return cast(_TeamIndex, self._indexes.team_model) # cast-ok: (team, public name) to positions + + @_core_team_model_to_deployment_indices.setter + def _core_team_model_to_deployment_indices(self, value: object) -> None: + self._indexes.assign("team_model", value) + + @property + def _core_model_names(self) -> set[str]: # mutable-ok: the live group set + self._indexes.refresh() + return cast(set[str], self._indexes.model_names) # cast-ok: group names + + @_core_model_names.setter + def _core_model_names(self, value: object) -> None: + self._indexes.assign("model_names", value) + + @property + def _core_deployment_names(self) -> list[str]: # mutable-ok: the live name list + self._indexes.refresh() + return cast(list[str], self._indexes.deployment_names) # cast-ok: litellm model names + + @_core_deployment_names.setter + def _core_deployment_names(self, value: object) -> None: + self._indexes.assign("deployment_names", value) + + @property + def _core_team_public_model_names(self) -> frozenset[str]: + self._indexes.refresh() + return self._indexes.team_public_model_names + + @_core_team_public_model_names.setter + def _core_team_public_model_names(self, value: object) -> None: + self._indexes.assign("team_public_model_names", value) + + @property + def _core_provider_default_deployment_ids(self) -> list[str]: # mutable-ok: the live id list + self._indexes.refresh() + return cast(list[str], self._indexes.provider_default_ids) # cast-ok: deployment ids + + @_core_provider_default_deployment_ids.setter + def _core_provider_default_deployment_ids(self, value: object) -> None: + self._indexes.assign("provider_default_ids", value) + + @property + def _core_pattern_router(self) -> PatternMatchRouter: + self._indexes.refresh() + return self._indexes.pattern_router + + @_core_pattern_router.setter + def _core_pattern_router(self, value: object) -> None: + self._indexes.pattern_router.patterns = getattr(value, "patterns", {}) # pyright: ignore[reportAttributeAccessIssue] # a router a caller assigned + + @property + def _core_team_pattern_routers(self) -> dict[str, PatternMatchRouter]: # mutable-ok: the live per-team routers + self._indexes.refresh() + return cast(dict[str, PatternMatchRouter], self._indexes.team_pattern_routers) # cast-ok: team id to its router + + @_core_team_pattern_routers.setter + def _core_team_pattern_routers(self, value: object) -> None: + routers: Final = as_str_mapping(value) or {} + self._indexes.write( + { + "team_patterns": [ + [ + str(team), + [ + [regex, deployments] + for regex, deployments in (as_str_mapping(getattr(router, "patterns", None)) or {}).items() + ], + ] + for team, router in routers.items() + ] + } + ) + + def replace_model_list(self, model_list: "list[UntypedEntry]") -> None: # mutable-ok: the list as assigned + """``router.model_list = [...]``: the list is swapped as given, without reindexing, and + router-core takes it whole.""" + self._core_model_list.replace(model_list) + self._registry_version += 1 + self._on_change() + + def _core_delete_deployment(self, id: str) -> UntypedEntry | None: + """Remove one deployment and return its stored entry (``_delete_deployment``).""" + try: + with self._deployment_index_lock: + deployment_idx: Final = self.model_id_to_deployment_index_map.get(id) + if deployment_idx is None: + return None + item: Final = self._core_model_list[deployment_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does + self._remove(id) + if self.checks is not None: + self.checks.unregister_deployment_budget(id) + if self.strategies is not None: + self.strategies.release_entry(item) + except Exception: # noqa: BLE001 # the core answers None for any failed delete + return None + self._on_change() + return item + + def _core_get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None": + """The router's deployment-budget callback (``registry().plugins("budget_limiter")``).""" + return self.checks.budget_limiter() if self.checks is not None else None + + @property + def _core_prompt_caching_pin_cache(self) -> DualCache: + """The prompt caching pins (``registry().plugins("prompt_caching_pins")``).""" + if self.checks is None: + return self.__getattr__("_core_prompt_caching_pin_cache") # pyright: ignore[reportReturnType] # raises RustCoreUnsupported + return self.checks.prompt_caching_pin_cache + + def replay_model_cost_registrations(self) -> None: + """Re-register every served deployment's cost-map entries (``replay_model_cost_registrations``).""" + provider_unresolved: Final = self._provider_unresolved_deployments + self._provider_unresolved_deployments = () + for create_deployment in provider_unresolved: + create_deployment() + for entry in tuple(self._core_model_list): + try: + deployment = Deployment(**entry) # pyright: ignore[reportArgumentType] # a stored entry + except Exception: # noqa: BLE001 # a malformed entry must not abort the rest of the rebuild + verbose_router_logger.exception( + "Router: could not rebuild cost-map entry for a deployment during a price data reload" + ) + continue + register_deployment_pricing(self._host, deployment) + self._invalidate_model_group_info_cache() + if provider_unresolved: + self._on_change() + + # -- one deployment --------------------------------------------------------------------------- + + def _core_has_model_id(self, candidate_id: str) -> bool: + handle: Final = self._live_handle() + key: Final = _index_key(candidate_id) + return handle is not None and key is not None and handle.has_model_id(key) + + def _core_get_model_info(self, id: str) -> UntypedEntry | None: + handle: Final = self._live_handle() + key: Final = _index_key(id) + idx: Final[object] = None if handle is None or key is None else handle.id_position(key) + return None if idx is None else self._core_model_list[idx] # pyright: ignore[reportArgumentType, reportCallIssue] # a position, or what a caller wrote, read as Python reads it + + def _core_get_deployment(self, model_id: str) -> Deployment | None: + entry: Final = self._core_get_model_info(model_id) + return None if entry is None else Deployment(**entry) # pyright: ignore[reportArgumentType] # a stored entry + + def _core_get_deployment_by_model_group_name(self, model_group_name: str) -> Deployment | None: + handle: Final = self._live_handle() + key: Final = _index_key(model_group_name) + indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) + model: Final = self._core_model_list[indices[0]] if indices else None # pyright: ignore[reportIndexIssue, reportArgumentType, reportCallIssue] # positions, or what a caller wrote, read as Python reads them + return None if model is None else Deployment(**model) # pyright: ignore[reportArgumentType] # a stored entry + + def _at(self, indices: object) -> tuple[UntypedEntry, ...]: + """The entries at ``indices`` (what an index holds), read as Python's list reads them.""" + return tuple(self._core_model_list[idx] for idx in indices) # pyright: ignore[reportGeneralTypeIssues, reportArgumentType, reportCallIssue] # positions, or what a caller wrote + + def _core_deployments_named(self, model_name: str) -> tuple[UntypedEntry, ...] | None: + handle: Final = self._live_handle() + key: Final = _index_key(model_name) + indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) + return None if indices is None else self._at(indices) + + def team_public_model_names_of(self, team_id: object) -> frozenset[str]: + handle: Final = self._live_handle() + if handle is None or not isinstance(team_id, str): + return frozenset() + return frozenset(handle.team_public_names_of(team_id)) + + def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[UntypedEntry, ...] | None: + handle: Final = self._live_handle() + team: Final = _index_key(team_id) + public: Final = _index_key(public_model_name) + indices: Final[object] = ( + None if handle is None or team is None or public is None else handle.team_positions(team, public) + ) + return None if indices is None else self._at(indices) + + # -- lookups (router-core `lookup.rs`): rows read at the stored entries -------------------- + + def _rows(self, rows: object) -> list[UntypedEntry]: # mutable-ok: a fresh list + """The stored entries router-core's lookup ``rows`` name, read as Python's list reads them: + a row it could not read is indexed here, and raises as the Python router's read raised.""" + entries: Final[list[UntypedEntry]] = [] # mutable-ok: filled per row + for row in cast(Sequence[Mapping[str, object]], rows): # cast-ok: router-core's lookup rows + if "raw" in row: + entries.append(self._core_model_list[row["raw"]]) # pyright: ignore[reportArgumentType, reportCallIssue] # what a caller wrote, read as Python reads it + elif "raw_positions" in row: + entries.extend(self._at(row["raw_positions"])) + elif "pattern" in row: + entries.append({**DeploymentTypedDict(**row["pattern"]), "model_name": row["name"]}) # pyright: ignore[reportCallIssue] # a pattern router's stored entry + else: + entries.append(self._row_entry(row)) + return entries + + def _row_entry(self, row: Mapping[str, object]) -> UntypedEntry: + held: Final[UntypedEntry] = self._core_model_list[int(str(row["at"]))] + name: Final = row.get("name") + named: Final = held if name is None else _renamed(held, str(name)) + group: Final = row.get("group") + if group is None: + return named + return _as_routing_group_row( + apply_routing_group_priority(self._core_routing_groups[str(group)], str(row["member"]), named) # pyright: ignore[reportArgumentType] # a stored entry + ) + + def _raise_at(self, raw: object) -> None: + """Read the list at the row router-core could not read it at, which raises.""" + if raw is not None: + _ = self._rows([raw]) + + def _list_scope(self, model_name: str | None, *, only: str | None = None) -> Mapping[str, object]: + """What ``get_model_list`` lists beyond the deployments: the aliases (read here, where an + alias item that is no mapping or lacks ``hidden`` raises as the Python router's read + does), the callable routing groups and the wildcard requests for ``model_name``.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + aliases: Final[list[Mapping[str, str]]] = [] # mutable-ok: filled per alias + alias_map: Final = self._core_model_group_alias or {} + if model_name is None or model_name in alias_map: + items: Final[Sequence[tuple[str, object]]] = ( + [(model_name, alias_map[model_name])] if model_name is not None else list(alias_map.items()) + ) + for model_alias, model_value in items: + if isinstance(model_value, str): + aliases.append({"alias": model_alias, "target": model_value}) + elif isinstance(model_value, dict): + item = RouterModelGroupAliasItem(**model_value) # pyright: ignore[reportCallIssue, reportArgumentType] # an alias item dict + if item["hidden"] is True and model_name is None: + continue + aliases.append({"alias": model_alias, "target": item["model"]}) + groups: Final = [ + {"name": name, "members": list(group.models)} + for name in (self._core_routing_groups or {}) + if (group := self.get_routing_group(name)) is not None + ] + wildcard: Final = ( + [model_name, f"{inferred_provider(model_name)}/{model_name}"] if model_name is not None else [] + ) + return {"aliases": aliases, "groups": groups, "wildcard": wildcard, "only": only} + + def _lookup(self) -> "RouterCoreHandle": + handle: Final = self._live_handle() + if handle is None: + raise RuntimeError("the host registry's lookups run in its Rust core") + return handle + + # -- deployments by name ---------------------------------------------------------------------- + + @staticmethod + def _core_should_include_deployment( + model_name: str, model: Mapping[str, object], team_id: str | None = None + ) -> bool: + info: Final = _model_info_of(model) + if team_id is not None and info.get("team_id") == team_id and model_name == info.get("team_public_model_name"): + return True + if model_name is not None and model.get("model_name") == model_name: # pyright: ignore[reportUnnecessaryComparison] # the core checks it + model_team_id: Final = info.get("team_id") + return team_id is None or model_team_id is None or model_team_id == team_id + return False + + @staticmethod + def _core_deployment_usable_by_team(model: Mapping[str, object] | Deployment, team_id: str | None) -> bool: + info: Final[object] = ( + model.model_info.model_dump() if isinstance(model, Deployment) else model.get("model_info") + ) + owner: Final = _MODEL_INFO_ADAPTER.validate_python(info).get("team_id") if isinstance(info, Mapping) else None + return team_may_use_deployment(owner, team_id) + + def _core_deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[Mapping[str, object]]: + """The deployments ``model`` names for this caller: alias, then team-first, then global, then + admin-across-teams (``_core_deployments_for_request``).""" + from litellm.litellm_core_utils.routing_common_utils import get_request_team_id, is_proxy_admin_request + + registered: Final = self._core_get_model_from_alias(model=model) or model + team_id: Final = get_request_team_id(request_kwargs) + deployments: Final = self._core_get_all_deployments(model_name=registered, team_id=team_id) + if deployments or team_id is not None or not is_proxy_admin_request(request_kwargs): + return deployments + return self._team_deployments_across_teams(registered) + + def _team_deployments_across_teams(self, model: str) -> list[UntypedEntry]: # mutable-ok: a fresh list + """Every team's deployments under public name ``model``, for a proxy admin calling without a team.""" + rows, teams = self._lookup().lookup_across_teams(model) + team_deployments: Final = self._rows(rows) + if teams > 1: + raise litellm_exceptions.BadRequestError( + message=( + f"Model name '{model}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=model, + llm_provider="", + ) + return team_deployments + + def _core_get_all_deployments( + self, model_name: str, model_alias: str | None = None, team_id: str | None = None + ) -> list[UntypedEntry]: # mutable-ok: a fresh list, as the core returns it + key: Final = _index_key(model_name) + return [] if key is None else self._rows(self._lookup().lookup_all(key, team_id, model_alias)) + + def get_routing_group(self, model_name: str) -> RoutingGroup | None: + """The routing group callable as ``model_name``: a deployment model_name or an alias of the + same name shadows it.""" + if not self._core_routing_groups: + return None + group: Final = self._core_routing_groups.get(model_name) + handle: Final = self._live_handle() + if ( + group is None + or (handle is not None and handle.named_positions(model_name) is not None) + or model_name in (self._core_model_group_alias or {}) + ): + return None + return group + + def _core_routing_group_has_alternatives(self, model_group: str | None) -> bool: + """A callable routing group whose member union spans more than one deployment.""" + if model_group is None: + return False + resolved: Final = self._core_get_model_from_alias(model=model_group) or model_group + group: Final = self.get_routing_group(resolved) + if group is None: + return False + handle: Final = self._lookup() + + def held(member: str) -> Sequence[object]: + return cast(Sequence[object], handle.named_positions(member) or ()) # cast-ok: positions, or junk + + return sum(len(held(member)) for member in group.models) > 1 + + def _core_get_model_list_from_model_alias( + self, model_name: str | None = None + ) -> list[UntypedEntry]: # mutable-ok: a fresh list + return self._rows( + self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="aliases")) + ) + + def _core_get_model_list_from_routing_groups(self, model_name: str | None = None) -> Sequence[UntypedEntry]: + """Callable routing groups as model-list rows: each member deployment under the group's name.""" + if model_name is None and self._routing_group_rows is not None: + return self._routing_group_rows + rows: Final = tuple( + self._rows(self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="groups"))) + ) + if model_name is None: + self._routing_group_rows = rows + return rows + + def _core_get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> list[UntypedEntry] | None: # mutable-ok: a fresh list, as get_model_list returns + if model_name is not None and _index_key(model_name) is None: + # No index holds it, so `_get_wildcard_deployments` reads it: what the pattern routers raise. + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + return [ + *self._core_pattern_router.get_deployments_by_pattern(model=model_name), + *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), + ] + return self._rows(self._lookup().lookup_model_list(model_name, team_id, self._list_scope(model_name))) + + def _core_wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model reads the base routers + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_owned: Final = team_router.get_deployments_by_pattern(model=model) if team_router is not None else () + return team_owned or self._core_pattern_router.get_deployments_by_pattern(model=model) + rows: Final = self._lookup().lookup_wildcard_route( + model, team_id, [model, f"{inferred_provider(model)}/{model}"] + ) + routed: Final = cast(Sequence[Mapping[str, object]], rows) # cast-ok: router-core's wildcard rows + return [cast(Mapping[str, object], row["pattern"]) for row in routed] # cast-ok: each carries its routed copy + + def _core_wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_owned: Final = ( + team_router.get_models_by_pattern(model=model) if team_router is not None else frozenset[str]() + ) + return team_owned or self._core_pattern_router.get_models_by_pattern(model=model) + + def _core_config_deployments(self) -> Iterator[Mapping[str, object]]: + """A generator, as the Python core's: the rows are read when iteration starts.""" + yield from self._rows(self._lookup().lookup_config()) + + # -- names ------------------------------------------------------------------------------------ + + def _core_get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: a fresh list + names, raw = self._lookup().lookup_model_names(team_id, self._list_scope(None)) + self._raise_at(raw) + return names + + def _core_get_model_ids( + self, model_name: str | None = None, exclude_team_models: bool = False + ) -> list[str]: # mutable-ok: a fresh list + if model_name is not None and _index_key(model_name) is None: + return [] + ids, raw = self._lookup().lookup_model_ids(model_name, exclude_team_models) + self._raise_at(raw) + return ids + + # -- resolving a name ------------------------------------------------------------------------- + + def _core_get_model_from_alias(self, model: str) -> str | None: + return resolve_model_group_alias(self._core_model_group_alias, model) + + def _core_routable_model_group(self, model_name: str) -> str: + target: Final = self._core_get_model_from_alias(model_name) + return target if target is not None else model_name + + def _core_is_recognized_model(self, model: str) -> bool: + handle: Final = self._live_handle() + key: Final = _index_key(model) + return ( + (handle is not None and key is not None and handle.has_model_name(key)) + or self._core_has_model_id(model) + or model in self._core_model_group_alias + or self.get_routing_group(model) is not None + ) + + def _core_get_model_group(self, id: str) -> list[UntypedEntry] | None: # mutable-ok: a fresh list + model_info: Final = self._core_get_model_info(id=id) + if model_info is None: + return None + return self._core_get_model_list(model_name=str(model_info["model_name"])) + + def _core_resolve_model_name_from_model_id( + self, model_id: str | None, custom_llm_provider: str | None = None + ) -> str | None: + if not model_id: + return None + handle: Final = self._lookup() + if handle.has_model_name(model_id): + return model_id + if self._core_has_model_id(model_id): + deployment: Final = self._core_get_deployment(model_id=model_id) + if deployment is not None and deployment.model_name: + return deployment.model_name + return model_id + listed, serving, raw = handle.lookup_name_serving(model_id, self._list_scope(None)) + self._raise_at(raw) + if not listed: + return None + if serving is not None: + return serving + if self._core_is_recognized_model(model_id): + return model_id + if custom_llm_provider is None: + return None + if self._core_pattern_router.routes_to_provider(model_id, custom_llm_provider): + return model_id + provider_prefixed: Final = f"{custom_llm_provider}/{model_id}" + if not model_id.startswith(f"{custom_llm_provider}/") and self._core_pattern_router.routes_to_provider( + provider_prefixed, custom_llm_provider + ): + return provider_prefixed + return None + + def _get_deployment_by_litellm_model(self, model: object) -> list[UntypedEntry]: # mutable-ok: a fresh list + handle: Final = self._live_handle() + if handle is None or not isinstance(model, str): + return [] + return list(self._at(handle.litellm_model_positions(model))) + + # -- credentials ------------------------------------------------------------------------------ + + @staticmethod + def _core_is_deployment_blocked(deployment: Deployment) -> bool: + model_info: Final[object] = getattr(deployment, "model_info", None) + return model_info is not None and getattr(model_info, "blocked", None) is True + + def _core_get_deployment_credentials(self, model_id: str) -> UntypedEntry | None: + deployment: Final = self._core_get_deployment(model_id=model_id) + if deployment is None or self._core_is_deployment_blocked(deployment): + return None + return CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) + + def _get_model_group_deployment_usable_by_team( + self, model_group_name: str, team_id: str | None + ) -> Deployment | None: + if _index_key(model_group_name) is None: + return None + position, raw = self._lookup().lookup_group_usable_by_team(model_group_name, team_id) + self._raise_at(raw) + return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry + + def _get_team_public_name_deployment(self, model_id: str, team_id: str | None) -> Deployment | None: + if team_id is None or _index_key(model_id) is None: + return None + position, raw = self._lookup().lookup_team_public_name(model_id, team_id) + self._raise_at(raw) + return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry + + def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None: + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_models: Final = team_router.route(model_id) if team_router else None + global_models: Final = tuple( + model + for model in (self._core_pattern_router.route(model_id) or ()) + if self._core_deployment_usable_by_team(model, team_id) + ) + candidates: Final = team_models or global_models + if not candidates: + return None + return Deployment(**candidates[0]) # a pattern router holds stored entry dicts + + def _core_get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None: + deployment: Final = ( + self._core_get_deployment(model_id=model_id) + or self._get_model_group_deployment_usable_by_team(model_group_name=model_id, team_id=team_id) + or self._get_team_public_name_deployment(model_id=model_id, team_id=team_id) + or self._get_wildcard_deployment_usable_by_team(model_id=model_id, team_id=team_id) + ) + if deployment is None or self._core_is_deployment_blocked(deployment): + return None + return deployment + + def _core_get_deployment_credentials_with_provider( + self, model_id: str, team_id: str | None = None + ) -> UntypedEntry | None: + deployment: Final = self._core_get_credential_deployment(model_id=model_id, team_id=team_id) + if deployment is None: + return None + base: Final = CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) + credential_name: Final = deployment.litellm_params.litellm_credential_name + resolved: Final[UntypedEntry] = dict(base) + if credential_name is not None: + credential_values: Final = CredentialAccessor.get_credential_values(credential_name) + if not credential_values: + verbose_router_logger.warning("Credential '%s' not found in credential_list", credential_name) + resolved.update(credential_values) + _ = resolved.pop("litellm_credential_name", None) + model: Final = deployment.litellm_params.model + resolved["model"] = model + resolved["custom_llm_provider"] = ( + deployment.litellm_params.custom_llm_provider or (model.split("/")[0] if "/" in model else None) or "openai" + ) + return resolved + + # -- routable names --------------------------------------------------------------------------- + + def _routable_deployments(self, model_name: str, team_id: str | None) -> tuple[UntypedEntry, ...]: + return tuple(self._rows(self._lookup().lookup_routable(self._core_routable_model_group(model_name), team_id))) + + def _core_team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": + """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, + else None (``team_ptu_ceiling``), kept until the model list next changes.""" + from litellm.litellm_core_utils.ptu_shares import ptu_share_teams, team_ptu_ceiling + + listed: Final = tuple(self._core_get_model_list() or ()) + if self._ptu_cache is None: + self._ptu_cache = (ptu_share_teams(listed), {}) + share_teams, ceilings = self._ptu_cache + if team_id not in share_teams: + return None + key: Final = (team_id, model_group) + if key not in ceilings: + ceilings[key] = team_ptu_ceiling( + listed, + self._core_model_list, + team_id, + model_group, + lambda name: self._core_wildcard_route_deployments(name, team_id), + ) + return ceilings[key] + + def _get_client( + self, deployment: Mapping[str, object], kwargs: Mapping[str, object], client_type: str | None = None + ) -> object: + """The SDK client cached in the routing cache for a deployment (``_get_client``); a pod + discovery deployment has none. ``max_parallel_requests`` is the deployment's slot limit, + built on first use.""" + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + from litellm.router_rust.core import RustCoreUnsupported + + # `deployment["model_info"]["id"]`: the Python router reads the id first, raising for a + # deployment without one whatever the client type. + _ = deployment["model_info"]["id"] # pyright: ignore[reportIndexIssue] # read as the Python router reads it + if client_type == "max_parallel_requests": + limit_lookup: Final = self.max_parallel_requests_lookup + if limit_lookup is None: + raise RustCoreUnsupported( + "registry().client(client_type='max_parallel_requests') is not served by the Rust router core" + ) + return limit_lookup(deployment, kwargs) + if _params_of(deployment).get("kubernetes_pod_discovery") is True: + return None + model_id: Final = _model_info_of(deployment).get("id") + span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) + stream: Final = "_stream" if kwargs.get("stream") is True else "" + with service_target(ROUTER_CLIENTS_TARGET): + if client_type == "async": + return self._core_cache.get_cache( + key=f"{model_id}{stream}_async_client", local_only=True, parent_otel_span=span + ) # pyright: ignore[reportUnknownMemberType] # DualCache's reads are untyped + return self._core_cache.get_cache(key=f"{model_id}{stream}_client", parent_otel_span=span) # pyright: ignore[reportUnknownMemberType] # see above + + def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: + """Per routable deployment: its ``supports_search_tool`` when a bool, else True when its route + builds an Anthropic Messages request, else None (``get_search_tool_support``).""" + return tuple(_search_tool_support(deployment) for deployment in self._routable_deployments(model_name, team_id)) + + def _core_get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: + """The deployment ids that could serve ``model`` for ``team_id``: the alias target, then a + routing group, then a team public name or wildcard route for a name that is no model + group, then the model group (``get_candidate_model_ids_for_route``). Read-only.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + resolved: Final = self._core_get_model_from_alias(model=model) or model + group: Final = self.get_routing_group(resolved) if self._core_routing_groups else None + ids, raw = self._lookup().lookup_candidate_ids( + resolved, + team_id, + None if group is None else list(group.models), + [resolved, f"{inferred_provider(resolved)}/{resolved}"], + ) + self._raise_at(raw) + return frozenset(ids) + + def _core_team_model_has_alternatives(self, deployment_id: str) -> bool: + """Whether a team deployment's public name has another unpaused deployment of that team.""" + found, raw = self._lookup().lookup_team_model_has_alternatives(deployment_id) + self._raise_at(raw) + return found + + def _core_resolve_served_model_name(self, model_name: str, served_model: str) -> str: + """``served_model`` when ``model_name`` routes to it (the alias target, a routing group + member, or the name itself), else ``model_name``.""" + target: Final = self._core_get_model_from_alias(model_name) or model_name + group: Final = self.get_routing_group(target) + members: Final = group.models if group is not None else () + return served_model if served_model == target or served_model in members else model_name + + def _core_resolved_litellm_models( + self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False + ) -> tuple[str, ...]: + """The provider model strings a call to ``model_name`` can be served by (``resolved_litellm_models``).""" + listed: Final = self._core_get_model_list(model_name=model_name, team_id=team_id) or () + if use_base_model: + return tuple( + backend for entry in listed if (backend := cost_map_backend_model(Deployment.model_validate(entry))) + ) + return tuple(model for entry in listed if isinstance(model := _params_of(entry).get("model"), str) and model) + + def _core_get_routable_deployments( + self, model: str, request_kwargs: Mapping[str, object] = MappingProxyType({}), call_type: str | None = None + ) -> tuple[Mapping[str, object], ...]: + """The deployments routing would consider for ``request_kwargs``, as the Rust core resolves + them, the endpoint of ``call_type`` included.""" + lookup: Final = self.routable_lookup + if lookup is None: + from litellm.router_rust.core import RustCoreUnsupported + + raise RustCoreUnsupported("registry().get_routable_deployments is not served by the Rust router core") + return tuple(lookup(model, request_kwargs, call_type)) + + def _core_get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: + deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None) + if deployment is None: + return None + model: Final = _params_of(deployment).get("model") + return None if model is None else str(model) + + def _core_map_team_model( + self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None + ) -> str | None: + """A team's public name to its group, or a deployment for a model-less call (``map_team_model``).""" + models: Final = self._core_get_model_list(model_name=team_model_name, team_id=team_id) + if not models: + return None + if team_model_name is None and custom_llm_provider is not None: + return self._team_model_for_provider(models, team_id, custom_llm_provider) + for model in models: + info = _model_info_of(model) + if info.get("team_id") == team_id: + if team_model_name is None: + public = info.get("team_public_model_name") or model.get("model_name") + return None if public is None else str(public) + return team_model_name + return None + + @staticmethod + def _team_model_for_provider(models: Sequence[UntypedEntry], team_id: str, custom_llm_provider: str) -> str | None: + def public_name(deployment: UntypedEntry) -> str | None: + info: Final = _model_info_of(deployment) + if info.get("team_id") != team_id: + return None + name: Final = info.get("team_public_model_name") + return name if isinstance(name, str) and name else str(deployment["model_name"]) + + def provider(deployment: UntypedEntry) -> object: + return provider_for_generic_call(_params_of(deployment)) # pyright: ignore[reportArgumentType] # the deployment's params + + matching: Final = next( + (m for m in models if public_name(m) is not None and provider(m) == custom_llm_provider), None + ) + if matching is None: + return None + name: Final = public_name(matching) + if name is None: + return None + if all(provider(m) == custom_llm_provider for m in models if public_name(m) == name): + return name + matched_id: Final = _model_info_of(matching).get("id") + return matched_id if isinstance(matched_id, str) else None + + # -- access groups ---------------------------------------------------------------------------- + + def _core_get_model_access_groups( + self, model_name: str | None = None, model_access_group: str | None = None, team_id: str | None = None + ) -> Mapping[str, Sequence[str]]: + use_cache: Final = model_name is None and model_access_group is None and team_id is None + if use_cache and self._access_groups_cache is not None: + return self._access_groups_cache + pairs, raw = self._lookup().lookup_access_groups( + model_name, model_access_group, team_id, self._list_scope(model_name) + ) + self._raise_at(raw) + groups: Final[dict[str, list[str]]] = {g: list(n) for g, n in pairs} # mutable-ok: as a defaultdict + if use_cache: + self._access_groups_cache = groups + return groups + + def _core_is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: + groups: Final = self._core_get_model_access_groups(model_access_group=model_access_group) + return any( + self._core_pattern_router.route(request=model) is not None for model in groups.get(model_access_group, []) + ) + + +# -- the registry() view ------------------------------------------------------------------------- + + +class RustRegistry(RouterRegistry): + """``registry()`` of the Rust core: ``RouterRegistry``'s queries over the host registry.""" + + __slots__ = () + + def includes( + self, + deployment: "DeploymentTypedDict | dict[str, object]", # mutable-ok: the RouterRegistry signature + *, + model_name: str | None = None, + team_id: str | None = None, + ) -> bool: + hosted: Final = self._router + if not isinstance(hosted, HostRegistry): + return super().includes(deployment, model_name=model_name, team_id=team_id) + if model_name is None: + return HostRegistry._core_deployment_usable_by_team(deployment, team_id) # pyright: ignore[reportPrivateUsage, reportArgumentType] # the view reads its own registry + return HostRegistry._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id) # pyright: ignore[reportPrivateUsage, reportArgumentType] # see above + + +def hosted_registry_view(hosted: HostRegistry) -> RouterRegistry: + """The ``registry()`` view over ``hosted``.""" + return RustRegistry(hosted) # pyright: ignore[reportArgumentType] # the host registry answers every _core_ query the view makes + + +# The registry indexes a deprecated attribute write replaces (``router.model_names = ...`` reaches +# the host registry as ``_core_model_names``), by the field router-core reads each one as. +INDEX_ATTRIBUTES: Final[Mapping[str, str]] = MappingProxyType( + { + "_core_model_names": "model_names", + "_core_model_name_to_deployment_indices": "model_name", + "_core_team_model_to_deployment_indices": "team_model", + } +) + + +def _positions(indices: object, length: int) -> list[int]: # mutable-ok: a JSON list + """The model-list positions ``indices`` read, as Python's ``model_list[index]`` reads them: a + negative one counts from the end, and one before the start becomes ``length``, a position past + the end, so the lookup raises ``IndexError`` as Python's does.""" + if not isinstance(indices, Iterable) or isinstance(indices, str | bytes): + return [] + return [ + index if index >= 0 else (index + length if index + length >= 0 else length) + for index in indices + if isinstance(index, int) and not isinstance(index, bool) + ] + + +def index_document(hosted: HostRegistry, written: Iterable[str]) -> Mapping[str, object]: + """The written registry indexes, as the host registry holds them now, in the shape router-core + reads (``IndexOverrides``): names a lookup finds, and the deployment positions each model name + (or ``(team_id, public name)`` pair) indexes. Entries Python could not read as such are left out.""" + document: Final[dict[str, object]] = {} # mutable-ok: built here + length: Final = len(hosted._core_model_list) # pyright: ignore[reportPrivateUsage] # positions read against the live list + for attribute in written: + value: object = getattr(hosted, attribute, None) + field: str = INDEX_ATTRIBUTES[attribute] + if field == "model_names": + names = value if isinstance(value, Iterable) and not isinstance(value, str | bytes | Mapping) else () + document[field] = sorted({name for name in names if isinstance(name, str)}) + elif field == "model_name": + entries = value if isinstance(value, Mapping) else {} + document[field] = { + name: _positions(indices, length) for name, indices in entries.items() if isinstance(name, str) + } + else: + pairs = value if isinstance(value, Mapping) else {} + document[field] = [ + [key[0], key[1], _positions(indices, length)] + for key, indices in pairs.items() + if isinstance(key, tuple) and len(key) == 2 and all(isinstance(part, str) for part in key) + ] + return document + + +DEFAULT_DEPLOYMENT_FAILING_ID: Final = "__litellm_default_deployment_failing_call__" +"""The id router-core serves a ``default_deployment`` under when every call to it fails.""" + +DEFAULT_DEPLOYMENT_COPIED_ID: Final = "__litellm_default_deployment_copied__" +"""The id router-core serves a callable ``default_deployment`` under when its ``model_info.id`` is +not a string: the host picks and calls Python's copy of it, so its id stays what was assigned.""" + + +def default_deployment_copy(value: object, model: str) -> object: + """``_try_early_resolve_deployments_for_model_not_in_names``' copy of the default deployment + for ``model``, raising what it raises for a value it cannot copy.""" + updated: Final = value.copy() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # any assigned value, read as Python reads it + updated["litellm_params"] = value["litellm_params"].copy() # pyright: ignore[reportIndexIssue, reportUnknownMemberType] # same + updated["litellm_params"]["model"] = model # pyright: ignore[reportUnknownMemberType] # same + return updated # pyright: ignore[reportUnknownVariableType] # same + + +def default_deployment_copy_error(value: object, model: str) -> BaseException: + """The error the default deployment's copy raises for ``model``.""" + try: + _ = default_deployment_copy(value, model) + except Exception as raised: # noqa: BLE001 # the error the pick raises + return raised + return RuntimeError(f"router-core: the default deployment copies for {model!r}") + + +def default_deployment_call_error(value: object) -> BaseException | None: + """The error every call to the default deployment raises, from its first reads + (``_update_kwargs_with_deployment``'s ``deployment["model_name"]``, ``_get_client``'s + ``deployment["model_info"]["id"]``); None when they read.""" + try: + served: Final = default_deployment_copy(value, "") + _ = served["model_name"] # pyright: ignore[reportIndexIssue] # read as Python reads it + _ = served["model_info"]["id"] # pyright: ignore[reportIndexIssue] # same + except Exception as raised: # noqa: BLE001 # the error the call raises + return raised + return None + + +def served_default_deployment(value: object) -> tuple[Mapping[str, object] | None, bool]: + """The ``default_deployment`` as router-core serves it, and whether resolving to it raises + (a value Python cannot copy). A deployment every call to fails is served under + ``DEFAULT_DEPLOYMENT_FAILING_ID``, one whose id is not a string under + ``DEFAULT_DEPLOYMENT_COPIED_ID``; names that are not strings are served as text.""" + if value is None: + return None, False + try: + copied: Final = default_deployment_copy(value, "") + except Exception: # noqa: BLE001 # any copy error: the pick raises it + return None, True + entry: Final = as_str_mapping(copied) or {} + model_info: Final = dict(as_str_mapping(entry.get("model_info")) or {}) + failing: Final = default_deployment_call_error(value) is not None + assigned_id: Final = model_info.get("id") + model_info["id"] = ( + DEFAULT_DEPLOYMENT_FAILING_ID + if failing + else assigned_id + if isinstance(assigned_id, str) + else DEFAULT_DEPLOYMENT_COPIED_ID + ) + name: Final = entry.get("model_name") + return { + **entry, + "model_name": name if isinstance(name, str) else str(name or ""), + "litellm_params": dict(as_str_mapping(entry.get("litellm_params")) or {}), + "model_info": model_info, + }, False diff --git a/litellm/router_rust/surface/replay.py b/litellm/router_rust/surface/replay.py new file mode 100644 index 00000000000..6c770e07883 --- /dev/null +++ b/litellm/router_rust/surface/replay.py @@ -0,0 +1,30 @@ +"""The Rust core's live routers, whose deployments a price data reload re-registers in +``litellm.model_cost`` until they retire (``_live_routers`` / ``_replay_live_router_model_cost``).""" + +from __future__ import annotations + +import weakref +from typing import TYPE_CHECKING, Final + +from litellm.utils import add_live_deployment_replay + +if TYPE_CHECKING: + from litellm.router_rust.surface.registry import HostRegistry + +_live_registries: Final[weakref.WeakSet[HostRegistry]] = weakref.WeakSet() + + +def track_live_registry(registry: HostRegistry) -> None: + _live_registries.add(registry) + + +def untrack_live_registry(registry: HostRegistry) -> None: + _live_registries.discard(registry) + + +def _replay_live_registries() -> None: + for registry in tuple(_live_registries): + registry.replay_model_cost_registrations() + + +add_live_deployment_replay(_replay_live_registries) diff --git a/litellm/router_rust/surface/retries.py b/litellm/router_rust/surface/retries.py new file mode 100644 index 00000000000..4a9c6429ffa --- /dev/null +++ b/litellm/router_rust/surface/retries.py @@ -0,0 +1,295 @@ +""" +The retry members ``RustCore`` serves: ``should_retry_this_error`` and +``get_num_retries_from_retry_policy``. router-core decides both; this module only describes the +Python exception the way an attempt failure is described and reads the answer back. +""" + +import dataclasses +import operator +from collections.abc import Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.types.router import RetryPolicy + +# The most retries a request's `num_retries` asks router-core for (its count is a u32). +_MOST_RETRIES: Final = 2**32 - 1 + +if TYPE_CHECKING: + from litellm.router_rust.core import Resolved + from litellm.rust_bridge._native import ( # pyright: ignore[reportPrivateUsage] # the stub's handle + RouterCoreHandle, + ) + + +def resolved_retry_policy(value: object) -> RetryPolicy | None: + """The router's ``retry_policy`` as the Python router's constructor keeps it: a dict is read + into a ``RetryPolicy``, a ``RetryPolicy`` is kept, anything else is no policy.""" + if isinstance(value, RetryPolicy): + return value + if isinstance(value, Mapping): + fields: Final[Mapping[str, int | None]] = value # pyright: ignore[reportUnknownVariableType] # the policy's fields + return RetryPolicy(**fields) # pyright: ignore[reportArgumentType] # validated by the model + return None + + +def _is_policy(entry: object) -> bool: + return isinstance(entry, RetryPolicy | Mapping) + + +def reads_as_policy(entry: object) -> bool: + """Whether ``resolve_retry_policy`` reads ``entry`` without raising: no entry, a ``RetryPolicy``, + or a mapping ``RetryPolicy(**entry)`` accepts.""" + if entry is None or isinstance(entry, RetryPolicy): + return True + if not isinstance(entry, Mapping): + return False + try: + _ = RetryPolicy(**entry) # pyright: ignore[reportUnknownArgumentType] # validated by the model + except Exception: # noqa: BLE001 # any error the lookup would raise + return False + return True + + +def is_policy_per_group(policies: object) -> bool: + """Whether ``model_group_retry_policy`` is what router-core's settings hold: a mapping of + group names to policies (or nothing).""" + if policies is None: + return True + if not isinstance(policies, Mapping): + return False + entries: Final[Mapping[object, object]] = policies # pyright: ignore[reportUnknownVariableType] # read as objects + return all(isinstance(group, str) and _is_policy(entry) for group, entry in entries.items()) + + +def readable_retry_policy(value: object) -> RetryPolicy | None: + """A router ``retry_policy`` written as a plain attribute, as router-core's settings hold it. + The attribute keeps what was written; a value the lookup raises for reaches router-core as + ``retry_policy_raises`` instead.""" + return resolved_retry_policy(value) if reads_as_policy(value) else None + + +@dataclasses.dataclass(frozen=True, slots=True) +class GroupPolicies: + """A ``model_group_retry_policy`` as router-core reads it, resolved the way + ``resolve_retry_policy`` reads it for each group the call may evaluate. + + ``document`` holds a group's policy where the lookup finds one, and an empty policy (which + grants nothing) where it finds ``None``; a group it does not find is absent, so the router's + ``retry_policy`` applies. ``raising`` names the groups whose lookup raises (an ``in``/``[]`` + the value does not support, or an entry that is not a policy, whose fields + ``get_num_retries_from_retry_policy`` then fails to read): the Python retry loop raises that + error in place of the attempt's on a hop's first failure. + """ + + document: Mapping[str, object] + raising: frozenset[str] + + +def _looked_up(policies: object, group: str) -> tuple[bool, object]: + """``(found, entry)`` as ``resolve_retry_policy`` looks ``group`` up; raises as it raises.""" + found: Final = group in policies # pyright: ignore[reportOperatorIssue] # any value, as the Python router reads it + return (True, policies[group]) if found else (False, None) # pyright: ignore[reportIndexIssue, reportUnknownVariableType] # see above + + +def _policy_document(entry: object) -> object: + """A readable entry as router-core reads it: no entry grants nothing, a mapping is read the way + ``RetryPolicy(**entry)`` reads it.""" + if entry is None: + return {} + if isinstance(entry, Mapping): + return RetryPolicy(**entry) # pyright: ignore[reportUnknownArgumentType] # checked by reads_as_policy + return entry + + +def group_policies(policies: object, groups: Iterable[str]) -> GroupPolicies: + """``policies`` resolved for ``groups`` (a mapping is read by its own keys).""" + if policies is None: + return GroupPolicies({}, raising=frozenset()) + if isinstance(policies, Mapping): + entries: Final[Mapping[object, object]] = policies # pyright: ignore[reportUnknownVariableType] # read as objects + # A key that is no string never matches a group name. + named: Final = {group: entry for group, entry in entries.items() if isinstance(group, str)} + readable: Final = {group: entry for group, entry in named.items() if reads_as_policy(entry)} + return GroupPolicies( + {group: _policy_document(entry) for group, entry in readable.items()}, + raising=frozenset(named) - frozenset(readable), + ) + document: Final[dict[str, object]] = {} # mutable-ok: built here; router-core reads it as plain JSON + raising: Final[set[str]] = set() # mutable-ok: the groups whose lookup raises + for group in groups: + try: + found, entry = _looked_up(policies, group) + except Exception: # noqa: BLE001 # the Python retry loop raises it on a hop's first failure + raising.add(group) + continue + if not found: + continue + if not reads_as_policy(entry): + raising.add(group) + continue + document[group] = _policy_document(entry) + return GroupPolicies(document, raising=frozenset(raising)) + + +def static_group_policies(policies: object) -> object: + """The router's ``model_group_retry_policy`` as router-core's static settings hold it. A value + that is not a mapping of policies is resolved per request instead (``group_policies``).""" + return group_policies(policies, ()).document if isinstance(policies, Mapping) else None # pyright: ignore[reportUnknownArgumentType] # read as objects + + +def raising_groups(request: Mapping[str, object], resolved: GroupPolicies) -> Sequence[str]: + """The groups whose ``model_group_retry_policy`` lookup raises on a hop's first failure, unless + the retry loop never reads it: the request opted out with ``num_retries=0``. A streamed Anthropic + messages call is no exception: only the stream's own retry rules read such an entry as no policy + (``_anthropic_messages_resolved_retry_policy``); a failure before the stream opened reaches the + retry loop.""" + if not resolved.raising or request.get("num_retries") == 0: + return () + return sorted(resolved.raising) + + +def policy_lookup_error(original: BaseException, model_group: str, retry_policy: object, policies: object) -> Exception: + """The error ``get_num_retries_from_retry_policy`` raises for ``model_group`` after ``original`` + failed (the Python retry loop raises it in the attempt's place, chained to it); a lookup that + unexpectedly reads is reported as such.""" + from litellm.litellm_core_utils.get_retry_from_policy import get_num_retries_from_retry_policy + + traceback: Final = original.__traceback__ + try: + # Looked up while `original` is being handled, as in the retry loop's `except` block, so + # the error chains to it the way Python chains it. + try: + raise original + except BaseException: # noqa: BLE001 # re-raised only to be handled here + _ = get_num_retries_from_retry_policy( + exception=original, # pyright: ignore[reportArgumentType] # any exception the hop raised + retry_policy=retry_policy, # pyright: ignore[reportArgumentType] # the router's resolved policy + model_group=model_group, + model_group_retry_policy=policies, # pyright: ignore[reportArgumentType] # read as the Python router reads it + ) + except Exception as raised: # noqa: BLE001 # the error the hop ends with + return raised + finally: + original.__traceback__ = traceback # rebind-ok: put back as the hop left it + return RuntimeError(f"the model_group_retry_policy lookup for {model_group!r} did not raise") + + +def router_policy_retries( + exception: Exception, retry_policy: object, policies: object, model_group: str | None +) -> int | None: + """``get_num_retries_from_retry_policy`` for a router ``model_group_retry_policy`` that is not + a mapping of policies, read as the Python router reads it (raising as it raises).""" + from litellm.litellm_core_utils.get_retry_from_policy import get_num_retries_from_retry_policy + + return get_num_retries_from_retry_policy( + exception=exception, + retry_policy=retry_policy, # pyright: ignore[reportArgumentType] # the router's resolved policy + model_group=model_group, + model_group_retry_policy=policies, # pyright: ignore[reportArgumentType] # read as the Python router reads it + ) + + +def request_retry_fields(value: object) -> Mapping[str, object]: + """A request's ``num_retries`` as router-core reads it. ``async_function_with_retries`` never + casts it: it compares it with 0, and counts with ``range`` only once it is positive.""" + if value is None: + return {} + if isinstance(value, int): + count: Final = int(value) + return {"num_retries": min(count, _MOST_RETRIES)} if count >= 0 else {"uncounted_retries": "negative"} + try: + if value == 0: + return {"num_retries": 0} + if not value > 0: # pyright: ignore[reportOperatorIssue] # any value, compared as the retry loop compares it + return {"uncounted_retries": "not_positive"} + return {"num_retries": min(operator.index(value), _MOST_RETRIES)} # pyright: ignore[reportArgumentType] # see above + except Exception: # noqa: BLE001 # the retry loop raises it on a hop's first failure + return {"uncounted_retries": "raises"} + + +def retry_count_error(original: BaseException, value: object) -> Exception: + """The error the retry loop raises reading a request's ``num_retries`` after ``original`` + failed, chained to it; a value that unexpectedly reads is reported as such.""" + traceback: Final = original.__traceback__ + try: + try: + raise original + except BaseException: # noqa: BLE001 # re-raised only to be handled here + if value > 0: # pyright: ignore[reportOperatorIssue] # `if num_retries > 0:` + _ = range(value) # pyright: ignore[reportArgumentType] # `for current_attempt in range(num_retries):` + except Exception as raised: # noqa: BLE001 # the error the hop ends with + return raised + finally: + original.__traceback__ = traceback # rebind-ok: put back as the hop left it + return RuntimeError(f"the request's num_retries {value!r} did not raise") + + +def _count(deployments: object) -> int: + return len(deployments) if isinstance(deployments, list) else 0 # pyright: ignore[reportUnknownArgumentType] # only the length is read + + +def _filled(fallbacks: object) -> bool: + return isinstance(fallbacks, Sequence) and len(fallbacks) > 0 # pyright: ignore[reportUnknownArgumentType] # only the length is read + + +class RustRetries: + """Retry members read from the Rust core's handle.""" + + _handle: "RouterCoreHandle" + _resolved: "Resolved" + + @property + def _written_retry_policy(self) -> object: + """The ``retry_policy`` attribute as written (``Resolved`` holds what router-core reads).""" + raise NotImplementedError + + def should_retry_this_error( + self, + error: Exception, + healthy_deployments: list[object] | None = None, # mutable-ok: the Python router's signature + all_deployments: list[object] | None = None, # mutable-ok: same + context_window_fallbacks: list[object] | None = None, # mutable-ok: same + content_policy_fallbacks: list[object] | None = None, # mutable-ok: same + regular_fallbacks: list[object] | None = None, # mutable-ok: same + ) -> bool: + """``True`` when the error is retried in the same group; otherwise the error is raised.""" + from litellm.router_rust.core import attempt_failure + + retried: Final = self._handle.should_retry( + dict(attempt_failure(error)), + (_count(healthy_deployments), _count(all_deployments)), + (context_window_fallbacks is not None, content_policy_fallbacks is not None, _filled(regular_fallbacks)), + ) + if not retried: + raise error + return True + + def get_num_retries_from_retry_policy(self, exception: Exception, model_group: str | None = None) -> int | None: + from litellm.router_rust.core import attempt_failure + + resolved: Final = self._resolved + written: Final = self._written_retry_policy + if not is_policy_per_group(resolved.model_group_retry_policy) or not reads_as_policy(written): + return router_policy_retries(exception, written, resolved.model_group_retry_policy, model_group) + # A dict lookup hashes any key and only a string can match; the native call takes strings alone. + _ = hash(model_group) + return self._handle.retry_policy_retries( + dict(attempt_failure(exception)), model_group if isinstance(model_group, str) else None + ) + + +__all__ = [ + "GroupPolicies", + "RustRetries", + "group_policies", + "is_policy_per_group", + "policy_lookup_error", + "raising_groups", + "readable_retry_policy", + "reads_as_policy", + "request_retry_fields", + "resolved_retry_policy", + "retry_count_error", + "router_policy_retries", + "static_group_policies", +] diff --git a/litellm/router_rust/surface/settings.py b/litellm/router_rust/surface/settings.py new file mode 100644 index 00000000000..c8635593de5 --- /dev/null +++ b/litellm/router_rust/surface/settings.py @@ -0,0 +1,236 @@ +""" +The Rust core's settings semantics on the host side: how ``ChangeSet.settings`` is read (the rules +of ``update_settings``), the ``settings().as_dict()`` view (``get_settings``), the router attributes +the Python core keeps as plain attributes, deleted settings, and the construction-time side effects +of the logging arguments. + +The rules are the Python router core's, ported (never imported). Settings change only through a +changeset; a direct attribute write is turned into one by ``RustCore.__setattr__``. +""" + +import logging +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger +from litellm.constants import RUNTIME_UPDATABLE_ROUTER_SETTINGS +from litellm.litellm_core_utils.routing_groups import validate_routing_strategy +from litellm.types.router import RetryPolicy, RoutingStrategy + +if TYPE_CHECKING: + from litellm.router_config import RouterConfig + from litellm.router_python_host import PythonHost + +MISSING: Final = object() + +# `update_settings` casts these with `int(value)`. +_INT_SETTINGS: Final = frozenset({"timeout", "num_retries", "retry_after", "allowed_fails", "cooldown_time"}) +CONTINUATION_SETTING: Final = "enable_mid_stream_fallback_continuation" + +# Settings the Python core keeps as plain router attributes and `update_settings` sets as given, +# with their constructor defaults. ``max_retries`` has none: it exists once a changeset sets it. +PLAIN_DEFAULTS: Final[Mapping[str, object]] = MappingProxyType( + { + "retry_policy": None, + "model_group_retry_policy": {}, + "enable_weighted_failover": False, + "fallback_on_unsupported_params": True, + } +) +# `get_settings`: the settings it lists, in its order (`routing_groups` comes last). +_VIEW_ORDER: Final = ( + "routing_strategy_args", + "routing_strategy", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "model_group_retry_policy", + "retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", + "enable_tag_filtering", + "tag_routing_prefix", + "fallback_on_unsupported_params", +) + +# Router attributes that are their constructor argument as given, else this default. Arguments +# the Rust core declines are only ever at their default here. +_ARGUMENT_ATTRIBUTES: Final[Mapping[str, object]] = MappingProxyType( + { + "set_verbose": False, + "debug_level": "INFO", + "client_ttl": 3600, + "stream_timeout": None, + "assistants_config": None, + "ignore_invalid_deployments": False, + "alerting_config": None, + "auto_router_capability_limit": None, + "deployment_affinity_ttl_seconds": 3600, + "session_affinity_repin_on_failover": False, + "model_group_affinity_config": None, + } +) +# Class-level and constant attributes of the Python core. +_CONSTANT_ATTRIBUTES: Final[Mapping[str, object]] = MappingProxyType( + {"default_cache_time_seconds": 1 * 60 * 60, "tenacity": None, "access_groups": None} +) + + +@dataclass(frozen=True) +class SettingsUpdate: + """What a ``ChangeSet.settings`` mapping changes, read with ``update_settings``' rules.""" + + values: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) # fields and plain settings + plain: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) # plain attributes, as stored + continuation: object = MISSING # the continuation flag, when given + routing_groups: object = MISSING # the routing groups input, when given + optional_pre_call_checks: object = MISSING # the optional pre-call checks, when given + failure: Exception | None = None # what stopped the reading; the settings before it stay applied + + +def read_settings_update(settings: Mapping[str, object], routing_strategy: object) -> SettingsUpdate: + """``_core_update_settings``' reading of each setting, in order: names that are not + runtime-updatable are ignored, the integer settings are cast with ``int``, a retry policy dict + becomes a ``RetryPolicy`` (any other type but ``None`` is dropped), and a strategy other than + ``routing_strategy``, the router's, is validated. The loop has no rollback: a setting that + raises ends it, and the ones before it are applied.""" + values: Final[dict[str, object]] = {} # mutable-ok: built here, then frozen into the update + plain: Final[dict[str, object]] = {} # mutable-ok: same + continuation: object = MISSING # rebind-ok: set when the flag is given + routing_groups: object = MISSING # rebind-ok: set when routing groups are given + optional_checks: object = MISSING # rebind-ok: set when the optional pre-call checks are given + failure: Exception | None = None # rebind-ok: set when a setting raises + try: + for name, value in settings.items(): + if name not in RUNTIME_UPDATABLE_ROUTER_SETTINGS: + verbose_router_logger.debug("Setting %s is not allowed", name) + elif name in _INT_SETTINGS: + values[name] = int(value) # pyright: ignore[reportArgumentType] # `int(kwargs[var])`, as given: a bad value raises int's own error + elif name == "routing_groups": + routing_groups = value + elif name == "optional_pre_call_checks": + # `set_optional_pre_call_checks` reads a list it is given as a set. + _ = frozenset[object](value) if value is not None else None # pyright: ignore[reportArgumentType] # as given: a bad value raises frozenset's own error + optional_checks = value + elif name == CONTINUATION_SETTING: + continuation = value + elif name == "retry_policy": + policy = RetryPolicy.model_validate(value) if isinstance(value, dict) else value + if policy is None or isinstance(policy, RetryPolicy): + plain[name] = policy + elif name in PLAIN_DEFAULTS or name == "max_retries": + plain[name] = value + else: + if name == "routing_strategy" and _normalized(value) != routing_strategy: + validate_routing_strategy(value) # pyright: ignore[reportArgumentType] # validated as given + values[name] = value + except Exception as raised: # noqa: BLE001 # raised by `apply` once the settings before it are applied + failure = raised # rebind-ok: None until a setting fails + return SettingsUpdate( + values=MappingProxyType(values), + plain=MappingProxyType(plain), + continuation=continuation, + routing_groups=routing_groups, + optional_pre_call_checks=optional_checks, + failure=failure, + ) + + +def _normalized(strategy: object) -> object: + """``_normalize_strategy``, which leaves anything but the enum as given.""" + return strategy.value if isinstance(strategy, RoutingStrategy) else strategy + + +def initial_plain(config: "RouterConfig") -> dict[str, object]: # mutable-ok: the core's own plain attributes + """The plain attributes as the constructor sets them: a ``retry_policy`` dict is read into a + ``RetryPolicy`` and any other type but a policy is no policy.""" + from litellm.router_rust.surface.retries import resolved_retry_policy + + plain: Final = {name: config.get(name, default) for name, default in PLAIN_DEFAULTS.items()} + plain["retry_policy"] = resolved_retry_policy(plain["retry_policy"]) + return plain + + +def settings_view( + values: Mapping[str, object], plain: Mapping[str, object], deleted: frozenset[str] +) -> dict[str, object]: # mutable-ok: the settings view is built fresh per call, as `get_settings` builds it + """``get_settings``: the listed settings the router has, then the routing groups. ``values`` + holds the settings fields and plain attributes the Rust core resolves.""" + view: Final[dict[str, object]] = {} # mutable-ok: see above + for name in _VIEW_ORDER: + if name in deleted: + continue + if name in plain: + view[name] = plain[name] + elif name in values: + view[name] = values[name] + view["routing_groups"] = values.get("routing_groups", []) + return view + + +def argument_attribute(config: "RouterConfig", name: str) -> object: + """A router attribute that is its constructor argument (or a constant), else ``MISSING``.""" + if name in _ARGUMENT_ATTRIBUTES: + return config.get(name, _ARGUMENT_ATTRIBUTES[name]) + if name in _CONSTANT_ATTRIBUTES: + return _CONSTANT_ATTRIBUTES[name] + return MISSING + + +def default_litellm_params(config: "RouterConfig") -> dict[str, object]: # mutable-ok: the router's own dict + """``default_litellm_params`` as the constructor completes it: the router timeout, no client + retries and the caching groups.""" + given: Final = _as_dict(config.get("default_litellm_params")) or {} + params: Final[dict[str, object]] = dict(given) # mutable-ok: see above + params.setdefault("timeout", config.get("timeout")) + params.setdefault("max_retries", 0) + metadata: Final = _as_dict(params.setdefault("metadata", {})) + if metadata is not None: + params["metadata"] = {**metadata, "caching_groups": config.get("caching_groups")} + return params + + +def _as_dict(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, dict) else None # pyright: ignore[reportUnknownVariableType] # the argument dict, as passed + + +def request_timeout(config: "RouterConfig") -> float | None: + """The per-attempt ``request_timeout``: only stored when a router timeout is also set.""" + from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout + + return get_configured_request_timeout() if config.get("timeout") is not None else None + + +def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None: + """What the constructor's logging arguments do: the host stops its debug info, and + ``set_verbose`` sets the router logger to ``debug_level``.""" + host.suppress_debug_info() + if config.get("set_verbose", False) is True: + level: Final = config.get("debug_level", "INFO") + if level == "INFO": + verbose_router_logger.setLevel(logging.INFO) + elif level == "DEBUG": + verbose_router_logger.setLevel(logging.DEBUG) + + +__all__ = [ + "CONTINUATION_SETTING", + "MISSING", + "PLAIN_DEFAULTS", + "SettingsUpdate", + "apply_construction_effects", + "argument_attribute", + "default_litellm_params", + "initial_plain", + "read_settings_update", + "request_timeout", + "settings_view", +] diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index 8e0c3d6a7e7..22159079f98 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -1,6 +1,11 @@ +import contextlib import logging -from collections.abc import Mapping, Sequence -from typing import Final +import threading +import time +import uuid +from collections.abc import Callable, Generator, Mapping, Sequence +from contextvars import ContextVar +from typing import Final, Protocol, TypeAlias from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -8,11 +13,31 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import log_redis_failure from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.types.utils import CallTypes IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60 +SELECTION_RESERVATION_TTL_SECONDS: Final = 30.0 +SELECTION_RESERVATION_MAX_KEYS: Final = 10_000 +_RESERVATIONS_LOCK: Final = threading.Lock() +_PICK_RESERVES: Final[ContextVar[bool]] = ContextVar("least_busy_pick_reserves") +_PICK_OWNER: Final[ContextVar[str | None]] = ContextVar("least_busy_pick_owner", default=None) + + +@contextlib.contextmanager +def picks_reserve(reserve: bool) -> Generator[None, None, None]: + token: Final = _PICK_RESERVES.set(reserve) + try: + yield + finally: + _PICK_RESERVES.reset(token) + + +def pick_reserves(default: bool) -> bool: + return _PICK_RESERVES.get(default) class _ModelInfo(TypedDict, total=False): @@ -21,10 +46,12 @@ class _ModelInfo(TypedDict, total=False): class _Metadata(TypedDict, total=False): model_group: ReadOnly[str | None] + model_info: ReadOnly[_ModelInfo | None] class _LitellmParams(TypedDict, total=False): metadata: ReadOnly[_Metadata | None] + litellm_metadata: ReadOnly[_Metadata | None] model_info: ReadOnly[_ModelInfo | None] @@ -32,6 +59,10 @@ class _CallKwargs(TypedDict, total=False): litellm_params: ReadOnly[_LitellmParams | None] +class CallIdentity(TypedDict, total=False): + litellm_call_id: ReadOnly[object] + + class _DeploymentModelInfo(TypedDict): id: ReadOnly[str | int] @@ -41,38 +72,85 @@ class _Deployment(TypedDict): _CALL_KWARGS: Final = TypeAdapter(_CallKwargs) +CALL_IDENTITY: Final = TypeAdapter(CallIdentity) _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment]) -_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None) +_MEMORY_COUNTS: Final[TypeAdapter[tuple[float | None, ...] | None]] = TypeAdapter(tuple[float | None, ...] | None) +_ATTEMPT_TIMES: Final[TypeAdapter[tuple[float, ...] | None]] = TypeAdapter(tuple[float, ...] | None) +_Pick: TypeAlias = tuple[float, str | None] +_PICKS: Final[TypeAdapter[tuple[_Pick, ...] | None]] = TypeAdapter(tuple[_Pick, ...] | None) +_PICKED_DEPLOYMENTS: Final[TypeAdapter[tuple[str, ...] | None]] = TypeAdapter(tuple[str, ...] | None) def _request_count_key(model_group: str, deployment_id: str) -> str: return f"{model_group}_request_count:{deployment_id}" +def _model_group(metadata: _Metadata | None) -> str | None: + return metadata.get("model_group") if metadata else None + + +def _deployment_id(model_info: _ModelInfo | None) -> str | int | None: + return model_info.get("id") if model_info else None + + +def _routed_deployment_id(metadata: _Metadata | None) -> str | int | None: + return _deployment_id(metadata.get("model_info")) if metadata else None + + def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None: - if is_batch_retrieve_call_type(kwargs.get("call_type")): + call_type: Final = kwargs.get("call_type") + if is_batch_retrieve_call_type(call_type) or call_type == CallTypes.arealtime.value: return None try: call: Final = _CALL_KWARGS.validate_python(kwargs) except ValidationError: return None - litellm_params: Final = call.get("litellm_params") - metadata: Final = litellm_params.get("metadata") if litellm_params else None - model_info: Final = litellm_params.get("model_info") if litellm_params else None - model_group: Final = metadata.get("model_group") if metadata else None - deployment_id: Final = model_info.get("id") if model_info else None + litellm_params: Final = call.get("litellm_params") or _LitellmParams() + routing_metadata: Final = (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) + model_group: Final = next((group for group in map(_model_group, routing_metadata) if group is not None), None) + deployment_id: Final = next( + ( + deployment_id + for deployment_id in ( + _deployment_id(litellm_params.get("model_info")), + *map(_routed_deployment_id, routing_metadata), + ) + if deployment_id is not None + ), + None, + ) if model_group is None or deployment_id is None: return None return model_group, str(deployment_id) -def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]: +def _reservation_key(deployment_id: str) -> str: + return f"least_busy_reservations:{deployment_id}" + + +def _started_attempts_key(call_id: str, ref: tuple[str, str]) -> str: + return f"least_busy_started:{call_id}:{_request_count_key(*ref)}" + + +def _picked_deployments_key(call_id: str) -> str: + return f"least_busy_picked:{call_id}" + + +def call_id_of(kwargs: Mapping[str, object]) -> str | None: + call_id: Final = kwargs.get("litellm_call_id") + return call_id if isinstance(call_id, str) else None + + +def _deployment_ids(healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]: return tuple( - _request_count_key(model_group, str(deployment["model_info"]["id"])) - for deployment in _DEPLOYMENTS.validate_python(healthy_deployments) + str(deployment["model_info"]["id"]) for deployment in _DEPLOYMENTS.validate_python(healthy_deployments) ) +def _request_count_keys(model_group: str, deployment_ids: tuple[str, ...]) -> tuple[str, ...]: + return tuple(_request_count_key(model_group, deployment_id) for deployment_id in deployment_ids) + + def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]: return tuple(0 if value is None else int(value) for value in values) @@ -84,12 +162,21 @@ def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]: return _as_counts(values) -def _least_busy( - healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...] -) -> Mapping[str, object] | None: - if not healthy_deployments: - return None - return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])] +def _unexpired(picks: Sequence[_Pick], now: float) -> tuple[_Pick, ...]: + return tuple(pick for pick in picks if now - pick[0] < SELECTION_RESERVATION_TTL_SECONDS) + + +def _without_one_pick(picks: tuple[_Pick, ...], call_id: str | None) -> tuple[_Pick, ...]: + tags: Final = tuple(tag for _, tag in picks) + owner: Final = next((tag for tag in (call_id, _PICK_OWNER.get()) if tag is not None and tag in tags), None) + if owner not in tags: + return picks + index: Final = tags.index(owner) + return (*picks[:index], *picks[index + 1 :]) + + +def _least_busy_index(counts: tuple[int, ...]) -> int: + return min(range(len(counts)), key=lambda index: counts[index]) def _warn_unreadable(model_group: str, error: Exception) -> None: @@ -111,24 +198,156 @@ def _warn_unwritable(key: str, error: Exception) -> None: ) +class _ReservationStore(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: tuple[object, ...], *, ttl: float) -> None: ... + + def delete_cache(self, key: str) -> None: ... + + +def _picks(store: _ReservationStore, deployment_id: str) -> tuple[_Pick, ...]: + return _PICKS.validate_python(store.get_cache(_reservation_key(deployment_id))) or () + + +def _attempt_times(store: _ReservationStore, key: str) -> tuple[float, ...]: + return _ATTEMPT_TIMES.validate_python(store.get_cache(key)) or () + + +def _picked_deployments(store: _ReservationStore, call_id: str) -> tuple[str, ...]: + return _PICKED_DEPLOYMENTS.validate_python(store.get_cache(_picked_deployments_key(call_id))) or () + + +def _put(store: _ReservationStore, key: str, values: tuple[object, ...], ttl: float) -> None: + # InMemoryCache keeps a live key's first TTL, so drop the key to restart it + store.delete_cache(key) + if values: + store.set_cache(key, values, ttl=ttl) + + +def _release_call_on(store: _ReservationStore, deployment_id: str, call_id: str) -> None: + kept: Final = tuple(pick for pick in _picks(store, deployment_id) if pick[1] != call_id) + _put(store, _reservation_key(deployment_id), kept, SELECTION_RESERVATION_TTL_SECONDS) + + +def release_unstarted_picks(store: _ReservationStore, call_id: str | None) -> None: + if call_id is None: + return + if not _picked_deployments(store, call_id): + return + with _RESERVATIONS_LOCK: + for deployment_id in frozenset(_picked_deployments(store, call_id)): + _release_call_on(store, deployment_id, call_id) + store.delete_cache(_picked_deployments_key(call_id)) + + +@contextlib.contextmanager +def owning_picks(store: _ReservationStore, call_id: str | None) -> Generator[None, None, None]: + owner: Final = call_id or _PICK_OWNER.get() or uuid.uuid4().hex + token: Final = _PICK_OWNER.set(owner) + try: + yield + finally: + _PICK_OWNER.reset(token) + release_unstarted_picks(store, owner) + + class LeastBusyLoggingHandler(CustomLogger): test_flag: bool = False logged_success: int = 0 logged_failure: int = 0 - def __init__(self, router_cache: DualCache): + def __init__( + self, + router_cache: DualCache, + clock: Callable[[], float] = time.monotonic, + reservation_store: _ReservationStore | None = None, + ): self.router_cache = router_cache self.router_cache_id = str(id(router_cache)) + self._clock = clock + self._reservations: Final[_ReservationStore] = ( + reservation_store + if reservation_store is not None + else InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS, clock=clock) + ) + + def _live_reservations(self, deployment_id: str, now: float) -> tuple[_Pick, ...]: + return _unexpired(_picks(self._reservations, deployment_id), now) + + def _store_reservations(self, deployment_id: str, picks: tuple[_Pick, ...]) -> None: + _put(self._reservations, _reservation_key(deployment_id), picks, SELECTION_RESERVATION_TTL_SECONDS) + + def _take_reservation(self, deployment_id: str, call_id: str | None, now: float) -> None: + self._store_reservations(deployment_id, _without_one_pick(self._live_reservations(deployment_id, now), call_id)) + + def _record_pick(self, call_id: str, deployment_id: str) -> None: + picked: Final = (*_picked_deployments(self._reservations, call_id), deployment_id) + _put(self._reservations, _picked_deployments_key(call_id), picked, SELECTION_RESERVATION_TTL_SECONDS) + + def _select( + self, + healthy_deployments: Sequence[Mapping[str, object]], + deployment_ids: tuple[str, ...], + keys: tuple[str, ...], + shared: tuple[int, ...] | None, + reserve: bool, + call_id: str | None, + ) -> Mapping[str, object] | None: + if not healthy_deployments: + return None + with _RESERVATIONS_LOCK: + now: Final = self._clock() + counts: Final = ( + self._this_workers_counts(keys) if shared is None else self._at_least_this_workers(keys, shared) + ) + live: Final = tuple(self._live_reservations(deployment_id, now) for deployment_id in deployment_ids) + index: Final = _least_busy_index(tuple(count + len(r) for count, r in zip(counts, live, strict=True))) + if reserve: + tag: Final = call_id if call_id is not None else _PICK_OWNER.get() + self._store_reservations(deployment_ids[index], (*live[index], (now, tag))) + if tag is not None: + self._record_pick(tag, deployment_ids[index]) + return healthy_deployments[index] + + def _start_attempt(self, ref: tuple[str, str], call_id: str | None) -> None: + with _RESERVATIONS_LOCK: + now: Final = self._clock() + self._take_reservation(ref[1], call_id, now) + self._count_locally(_request_count_key(*ref), 1) + if call_id is not None: + key: Final = _started_attempts_key(call_id, ref) + started: Final = (*_attempt_times(self._reservations, key), now) + _put(self._reservations, key, started, IN_FLIGHT_COUNT_TTL_SECONDS) + + def _end_attempt(self, ref: tuple[str, str], call_id: str | None) -> bool: + if call_id is None: + with _RESERVATIONS_LOCK: + self._count_locally(_request_count_key(*ref), -1) + return True + key: Final = _started_attempts_key(call_id, ref) + with _RESERVATIONS_LOCK: + started: Final = _attempt_times(self._reservations, key) + if started: + _put(self._reservations, key, started[1:], IN_FLIGHT_COUNT_TTL_SECONDS) + self._count_locally(_request_count_key(*ref), -1) + else: + self._take_reservation(ref[1], call_id, self._clock()) + return bool(started) @with_service_target("router_usage") def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: - self._increment(kwargs, 1) + ref: Final = _deployment_ref(kwargs) + if ref is None: + return + self._start_attempt(ref, call_id_of(kwargs)) + self._count_shared(_request_count_key(*ref), 1) @with_service_target("router_usage") def log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -136,7 +355,7 @@ class LeastBusyLoggingHandler(CustomLogger): def log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_failure += 1 @@ -144,7 +363,7 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -152,15 +371,20 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_failure += 1 @with_service_target("router_usage") def get_available_deployments( - self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]] + self, + model_group: str, + healthy_deployments: Sequence[Mapping[str, object]], + reserve: bool = False, + call_id: str | None = None, ) -> Mapping[str, object] | None: - keys: Final = _request_count_keys(model_group, healthy_deployments) + deployment_ids: Final = _deployment_ids(healthy_deployments) + keys: Final = _request_count_keys(model_group, deployment_ids) redis_cache: Final = self.router_cache.redis_cache if redis_cache is not None: try: @@ -168,15 +392,19 @@ class LeastBusyLoggingHandler(CustomLogger): except Exception as e: _warn_unreadable(model_group, e) else: - return _least_busy(healthy_deployments, shared) - local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) - return _least_busy(healthy_deployments, local) + return self._select(healthy_deployments, deployment_ids, keys, shared, reserve, call_id) + return self._select(healthy_deployments, deployment_ids, keys, None, reserve, call_id) @with_service_target("router_usage") async def async_get_available_deployments( - self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]] + self, + model_group: str, + healthy_deployments: Sequence[Mapping[str, object]], + reserve: bool = False, + call_id: str | None = None, ) -> Mapping[str, object] | None: - keys: Final = _request_count_keys(model_group, healthy_deployments) + deployment_ids: Final = _deployment_ids(healthy_deployments) + keys: Final = _request_count_keys(model_group, deployment_ids) redis_cache: Final = self.router_cache.redis_cache if redis_cache is not None: try: @@ -184,42 +412,51 @@ class LeastBusyLoggingHandler(CustomLogger): except Exception as e: _warn_unreadable(model_group, e) else: - return _least_busy(healthy_deployments, shared) - local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys) - return _least_busy(healthy_deployments, local) + return self._select(healthy_deployments, deployment_ids, keys, shared, reserve, call_id) + return self._select(healthy_deployments, deployment_ids, keys, None, reserve, call_id) - def _increment(self, kwargs: Mapping[str, object], delta: int) -> None: + def _settle(self, kwargs: Mapping[str, object]) -> None: ref: Final = _deployment_ref(kwargs) - if ref is None: + if ref is None or not self._end_attempt(ref, call_id_of(kwargs)): return - key: Final = _request_count_key(*ref) - redis_cache: Final = self.router_cache.redis_cache + self._count_shared(_request_count_key(*ref), -1) + + async def _async_settle(self, kwargs: Mapping[str, object]) -> None: + ref: Final = _deployment_ref(kwargs) + if ref is None or not self._end_attempt(ref, call_id_of(kwargs)): + return + await self._async_count_shared(_request_count_key(*ref), -1) + + def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]: + return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) + + def _at_least_this_workers(self, keys: tuple[str, ...], shared: tuple[int, ...]) -> tuple[int, ...]: + return tuple(max(counts) for counts in zip(shared, self._this_workers_counts(keys), strict=True)) + + def _count_locally(self, key: str, delta: int) -> None: try: local: Final = self.router_cache.increment_cache( key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS ) if local < 0: self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS) - if redis_cache is None: - return + except TypeError as e: + _warn_unwritable(key, e) + + def _count_shared(self, key: str, delta: int) -> None: + redis_cache: Final = self.router_cache.redis_cache + if redis_cache is None: + return + try: redis_cache.increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS) except Exception as e: _warn_unwritable(key, e) - async def _async_increment(self, kwargs: Mapping[str, object], delta: int) -> None: - ref: Final = _deployment_ref(kwargs) - if ref is None: - return - key: Final = _request_count_key(*ref) + async def _async_count_shared(self, key: str, delta: int) -> None: redis_cache: Final = self.router_cache.redis_cache + if redis_cache is None: + return try: - local: Final = await self.router_cache.async_increment_cache( - key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS - ) - if local is not None and local < 0: - await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS) - if redis_cache is None: - return await redis_cache.async_increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS) except Exception as e: _warn_unwritable(key, e) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 7a8a645ad13..03485ebd5f8 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -125,12 +125,13 @@ _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( def _first_present(*sources: Mapping[str, Any] | None, key: str) -> int | float | None: """Return *key* from the first source mapping where it's set, so callers can support a setting living in more than one deployment config location. Sources - are checked in order from most to least specific to that setting.""" + are checked in order from most to least specific to that setting. Only a number + counts: a bool or a string is skipped, so the router default applies.""" for source in sources: if source is None: continue value = source.get(key) - if value is not None: + if isinstance(value, int | float) and not isinstance(value, bool): return value return None @@ -811,7 +812,7 @@ def _is_allowed_fails_set_on_router( return False -def cast_exception_status_to_int(exception_status: str | int) -> int: +def cast_exception_status_to_int(exception_status: object) -> int: if isinstance(exception_status, str): try: exception_status = int(exception_status) @@ -820,6 +821,13 @@ def cast_exception_status_to_int(exception_status: str | int) -> int: "Unable to cast exception status to int %s. Defaulting to status=500.", exception_status ) exception_status = 500 + # A bool or a number no HTTP status can be is read as a 500, as the Rust router core reads it. + if ( + isinstance(exception_status, bool) + or not isinstance(exception_status, int) + or not 0 <= exception_status <= 65535 + ): + return 500 return exception_status diff --git a/litellm/router_utils/deployment_numbers.py b/litellm/router_utils/deployment_numbers.py new file mode 100644 index 00000000000..7cee96f6a64 --- /dev/null +++ b/litellm/router_utils/deployment_numbers.py @@ -0,0 +1,90 @@ +# pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false +"""Numbers a router deployment cannot be served with, refused the same way by both router cores. + +A whole number wider than 64 bits (a `ptu_shares` of `1e20`) does not fit the router's JSON +deployment record, and an infinite or NaN cost would read back as null there, so the call +would silently be priced from the model cost map. Both are refused when the deployment is +built or added, with the error naming the field. +""" + +import math +import re +from collections.abc import Iterator, Mapping +from typing import Final + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +# The whole numbers a deployment record holds (serde_json's i64 and u64). +SMALLEST_NUMBER: Final = -(2**63) +LARGEST_NUMBER: Final = 2**64 - 1 +# A pricing field: `input_cost_per_token`, `cache_read_input_token_cost`, ... +_COST_FIELD: Final = re.compile(r"cost_per|_cost$") + + +def _reads_as_non_finite(value: object) -> bool: + """Whether a cost, as written, is an infinity or a NaN (``float()`` reads ``"inf"`` and ``"1e400"``).""" + if isinstance(value, float): + return not math.isfinite(value) + if not isinstance(value, str): + return False + try: + return not math.isfinite(float(value)) + except ValueError: + return False + + +def _is_cost(key: object) -> bool: + return isinstance(key, str) and _COST_FIELD.search(key) is not None + + +def _number_leaves(value: object) -> Iterator[tuple[str, object, bool]]: + """Every value under ``value`` that is no container, in order: its path, and whether it sits in a + cost field. Each container is walked once, so an entry that holds itself ends.""" + pending: Final[list[tuple[object, str, bool]]] = [(value, "", False)] # mutable-ok: the walk's work list + walked: Final[set[int]] = set() # mutable-ok: the ids walked so far + while pending: + current, path, cost = pending.pop() + if not isinstance(current, Mapping | list | tuple): + yield path, current, cost + elif id(current) not in walked: + walked.add(id(current)) + pending.extend( + reversed( + [(item, f"{path}.{key}".removeprefix("."), cost or _is_cost(key)) for key, item in current.items()] + if isinstance(current, Mapping) + else [(item, f"{path}[{index}]", cost) for index, item in enumerate(current)] + ) + ) + + +def _refusal(entry: Mapping[str, object], reason: str) -> ValueError: + return ValueError(f"Deployment {entry.get('model_name')!r} cannot be served: {reason}") + + +def refuse_wide_numbers(entry: object) -> None: + """Refuse a deployment holding a whole number wider than 64 bits, by name.""" + deployment: Final = as_str_mapping(entry) + if deployment is None: + return + for path, value, _ in _number_leaves(deployment): + if isinstance(value, int) and not SMALLEST_NUMBER <= value <= LARGEST_NUMBER: + raise _refusal( + deployment, + f"{path} is {value}, outside what the router can hold " + f"({SMALLEST_NUMBER} to {LARGEST_NUMBER}). Use a smaller value.", + ) + + +def refuse_unservable_numbers(entry: object) -> None: + """``refuse_wide_numbers``, and a deployment with an infinite or NaN cost.""" + refuse_wide_numbers(entry) + deployment: Final = as_str_mapping(entry) + if deployment is None: + return + for path, value, cost in _number_leaves(deployment): + if cost and _reads_as_non_finite(value): + raise _refusal( + deployment, + f"{path} is {value!r}, and the router only prices with finite costs. " + "Set a finite cost, or remove it to use the model cost map's price.", + ) diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py index 4a87e582c97..7f58c138bba 100644 --- a/litellm/router_utils/endpoint_eligibility.py +++ b/litellm/router_utils/endpoint_eligibility.py @@ -57,10 +57,14 @@ class _EndpointDeclaration(BaseModel): return (value,) if isinstance(value, str) else value +def endpoint_for_call(call_type: object, module: object = None) -> str | None: + qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type + return CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None + + @contextmanager def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]: - qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type - endpoint: Final = CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None + endpoint: Final = endpoint_for_call(call_type, module) token: Final = _REQUEST_SELECTION.set((endpoint, model if isinstance(model, str) else None)) try: yield @@ -68,6 +72,10 @@ def endpoint_selection(call_type: object, module: object = None, model: object = _REQUEST_SELECTION.reset(token) +def selected_endpoint() -> str | None: + return _REQUEST_SELECTION.get()[0] + + def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None: try: declaration: Final = _EndpointDeclaration.model_validate(model_info) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e8f832b4b2d..20f9e114747 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -632,6 +632,8 @@ def get_fallback_model_group( ## check for specific model group-specific fallbacks for idx, item in enumerate(fallbacks): if isinstance(item, dict): + if not item: + continue # an empty entry names no group: the next entries still apply fallback_key = next(iter(item)) if fallback_key == model_group: # check exact match fallback_model_group = item[model_group] diff --git a/litellm/router_utils/global_callback_holders.py b/litellm/router_utils/global_callback_holders.py new file mode 100644 index 00000000000..53a7c20cff3 --- /dev/null +++ b/litellm/router_utils/global_callback_holders.py @@ -0,0 +1,53 @@ +""" +The routers of this process that hold optional pre-call checks and routing callbacks, whatever core +serves them. + +A router that retires hands the checks it vacates in its host's ``callbacks`` list to a surviving +router's same-keyed check (``routers_holding_global_callbacks``), and a router that drops a check at +runtime keeps it registered while another live router still has one of that type +(``live_check_owners``). Both rules look at every router in the process, so the sets live here, +outside any one router core. + +A member of ``routers_holding_global_callbacks`` serves ``_global_callback_candidates()``, +``_core_adaptive_routers`` and ``_sync_adaptive_router_hooks()``; a member of ``live_check_owners`` +serves ``optional_callbacks``. +""" + +import threading +import weakref +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final, Protocol + +if TYPE_CHECKING: + from litellm.integrations.custom_logger import CustomLogger + + +class GlobalCallbackHolder(Protocol): + """A router whose callbacks a retiring router may hand its vacated checks to.""" + + @property + def _core_adaptive_routers(self) -> Mapping[str, object]: ... + + def _global_callback_candidates(self) -> "tuple[CustomLogger, ...]": ... + + def _sync_adaptive_router_hooks(self) -> None: ... + + +class CheckOwner(Protocol): + """A live router with optional pre-call checks.""" + + @property + def optional_callbacks(self) -> "Sequence[object] | None": ... + + +routers_holding_global_callbacks: Final["weakref.WeakSet[GlobalCallbackHolder]"] = weakref.WeakSet() +routers_holding_global_callbacks_lock: Final = threading.Lock() +live_check_owners: Final["weakref.WeakSet[CheckOwner]"] = weakref.WeakSet() + +__all__ = [ + "CheckOwner", + "GlobalCallbackHolder", + "live_check_owners", + "routers_holding_global_callbacks", + "routers_holding_global_callbacks_lock", +] diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index c759af8fdb8..1c9cb10a48b 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -6,6 +6,7 @@ and exposes it for router candidate filtering. """ import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from typing_extensions import TypedDict @@ -78,7 +79,7 @@ class DeploymentHealthCache: merged: Final = { model_id: state for model_id, state in {**existing, **states}.items() - if isinstance(state, dict) and (now - state.get("timestamp", 0)) < expiry_seconds + if isinstance(state, dict) and (now - _timestamp_of(state)) < expiry_seconds # pyright: ignore[reportUnknownArgumentType] # a cached record } self.cache.set_cache( key=self.CACHE_KEY, @@ -101,7 +102,7 @@ class DeploymentHealthCache: for model_id, state in raw.items() if isinstance(state, dict) and not state.get("is_healthy", True) - and (now - state.get("timestamp", 0)) < self.staleness_threshold + and (now - _timestamp_of(state)) < self.staleness_threshold # pyright: ignore[reportUnknownArgumentType] # a cached record } @with_service_target(HEALTH_CHECKS_TARGET) @@ -129,3 +130,10 @@ class DeploymentHealthCache: str(e), ) return set() + + +def _timestamp_of(state: Mapping[str, object]) -> float: + """A health record's timestamp; one that is no number reads as 0 (stale), so a bad record + drops out alone instead of turning health routing off for every deployment.""" + timestamp: Final = state.get("timestamp", 0) + return float(timestamp) if isinstance(timestamp, int | float) and not isinstance(timestamp, bool) else 0.0 diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 954815f6a4f..dc170116089 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,6 +1,6 @@ from asyncio import Future from collections.abc import Mapping, Sequence -from typing import Never, final +from typing import Generic, Literal, Never, NotRequired, TypedDict, TypeVar, final import httpx from pydantic import JsonValue @@ -218,6 +218,8 @@ __all__ = [ "NativeDiagnosticProcessor", "ProcessReservedForForking", "ResponsesWebSocketConnection", + "RouterCall", + "RouterCoreHandle", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", @@ -296,3 +298,493 @@ class _SecretManagerRuntime: timeout: float | httpx.Timeout | None = None, primary_secret_name: str | None = None, ) -> Future[JsonValue]: ... + +# The FailureKind names an attempt outcome's `kind` takes (router-core `FailureKind`). +_FailureKind = Literal[ + "RateLimitError", + "ContextWindowExceededError", + "ContentPolicyViolationError", + "AuthenticationError", + "PermissionDeniedError", + "NotFoundError", + "BadRequestError", + "Timeout", + "APIConnectionError", + "InternalServerError", + "ServiceUnavailableError", + "BadGatewayError", + "GuardrailIntervention", + "Declined", + "CallerCredentialAuthenticationError", + "CallerCredentialRateLimitError", + "APIError", +] + +# One event a call reported since the last step (router-core `RouterEvent`), carried on the next step. +class _Event(TypedDict): + kind: str + +# Every step `RouterCall.next` returns carries the events reported since the last one. +class _Step(TypedDict): + events: list[_Event] + +class _AttemptStep(_Step): + kind: Literal["attempt"] + deployment_id: str + # The deployment as a `model_list` entry, with its id in `model_info`. + deployment: dict[str, JsonValue] + # The name this attempt serves: the requested model or the fallback target. + model: str + model_group: str + target_order: int | None + candidate_ids: list[str] + retry: int + fallback_depth: int + fallback_level: int + timeout: float | None + # Request params the fallback walk's dict targets merged in. + params: dict[str, JsonValue] | None + # The walk level whose request metadata the attempt carries. + scope: int + max_retries: int + model_group_size: int + +class _SleepStep(_Step): + kind: Literal["sleep"] + seconds: float + +class _AllowFallbackStep(_Step): + kind: Literal["allow_fallback"] + target: str + +class _RefuseFallbacksStep(_Step): + kind: Literal["refuse_fallbacks"] + # The walk level whose chain the dict targets are. + level: int + targets: list[dict[str, JsonValue]] + +class _SelectStep(_Step): + kind: Literal["select"] + stage: Literal[ + "plugins", + "strategy", + "tags", + "pre_call", + "text_only", + "callback_filters", + "routing_plugins", + "provider_request", + ] + strategy: str | None + model_group: str + # The name the request resolved from, which strategies key their state by. + model: str + # Whose selector a strategy reads: "default", "group" (with `group`) or "override". + scope: str + group: str | None + # The deployments the name resolved to, before health filters. + pool: list[str] + candidates: list[str] + # `routing_plugins` only: whether the plugins run. + run: NotRequired[bool] + # `callback_filters` only: the ids the request already failed on. + excluded_ids: NotRequired[list[str]] + skipped_ids: NotRequired[list[str]] + +# Pre-routing before a pick resolves `model`: answered with the model group to resolve instead +# (a string), a custom strategy's deployment document (a mapping), or None. +class _PreRouteStep(_Step): + kind: Literal["pre_route"] + model: str + fallback_depth: int + retry: int + +# One RouterCache op against the router's DualCache. `key` is set for every op but `batch_get`, +# and `increment_usage`, which set `keys`; `value` is the JSON to store (`set`), the amount +# (`increment`) or one amount per key (`increment_usage`); `ttl` is in seconds; `local` reads or +# writes this worker's in-memory tier only. Outcomes: get -> the JSON value or None; batch_get -> +# one JSON value or None per key, in order; increment -> the new int or float; set, delete and +# increment_usage -> None. +class _CacheStep(_Step): + kind: Literal["cache"] + op: Literal["get", "set", "increment", "increment_usage", "batch_get", "delete"] + key: str | None + keys: list[str] | None + value: JsonValue + ttl: float | None + local: bool + +# One lookup of a model-info query (router-core `ModelInfoLookup`): its `lookup` name and that +# lookup's arguments, answered as the lookup documents. +class _ModelInfoStep(_Step): + kind: Literal["model_info"] + lookup: Literal[ + "model_info", + "cost_entry", + "provider", + "pattern_models", + "bedrock_base", + "discovered", + "user_model_info", + "log", + "group_params", + "group_deployment", + "group_capabilities", + "group_seed", + "reasoning_intersect", + ] + model: NotRequired[JsonValue] + tolerate: NotRequired[bool] + exact: NotRequired[bool] + key: NotRequired[str] + process: NotRequired[bool] + name: NotRequired[str] + model_id: NotRequired[str] + id: NotRequired[JsonValue] + level: NotRequired[Literal["debug", "error"]] + message: NotRequired[str] + index: NotRequired[int] + model_info: NotRequired[JsonValue] + litellm_model: NotRequired[JsonValue] + llm_provider: NotRequired[JsonValue] + mapped: NotRequired[bool] + info: NotRequired[dict[str, JsonValue]] + current: NotRequired[JsonValue] + resolved: NotRequired[JsonValue] + +class _FailedStep(_Step): + kind: Literal["failed"] + error: str + error_kind: Literal[ + "config", + "unknown_model", + "no_healthy_deployments", + "attempt", + "cache", + "retired", + "paused", + "not_pass_through", + "no_pass_through_deployments", + "only_markers", + "ptu_reserved", + "context_window_exceeded", + "unsupported_params", + "routing_plugins_left_nothing", + "routing_plugins_need_async", + "multiple_teams", + "reserved", + "invalid_router_weights", + "non_numeric_weight", + "missing_routing_group", + "default_deployment_raises", + "index_out_of_range", + "model_info", + ] + attempt_failed: bool + status: int | None + failure_kind: _FailureKind | None + error_model: NotRequired[str] + error_detail: NotRequired[str] + error_params: NotRequired[list[str]] + # `model_info` only: the builtin exception the lookup raised. + error_exception: NotRequired[str] + +# The end of a `RouterCoreHandle.start` (and the stream recovery calls). +class _DoneStep(_Step): + kind: Literal["done"] + deployment_id: str + model_group: str + attempted_retries: int + max_retries: int + attempted_fallbacks: int + total_tokens: int + +# The end of a `RouterCoreHandle.select`. +class _SelectedStep(_Step): + kind: Literal["done"] + deployment_id: str + model_group: str + deployment: dict[str, JsonValue] + +# The end of a `RouterCoreHandle.healthy`. +class _HealthyStep(_Step): + kind: Literal["done"] + # The name was a deployment id. + single: bool + deployment_ids: list[str] + deployments: list[dict[str, JsonValue]] + +# The end of a `RouterCoreHandle.report_failure`. +class _ReportedStep(_Step): + kind: Literal["done"] + cooled: bool + # The cooldown applied, in seconds; None when the deployment was not cooled down. + cooldown_time: float | None + +# The end of a `RouterCoreHandle.report_success` or `count_success`. +class _CountedStep(_Step): + kind: Literal["done"] + key: str + +# One `os.environ/` reference a `secrets` step asks for: a top-level param (`key`), or an +# env-only `dataSources[source].parameters[key]` (`env_only`). +class _SecretRef(TypedDict): + name: str + key: str + env_only: bool + source: NotRequired[int] + +# The deployment build's `os.environ/` references: answered with one value per ref, in order. +class _SecretsStep(_Step): + kind: Literal["secrets"] + index: int + refs: list[_SecretRef] + +# The deployment build's steps for one deployment. `validate` and `added` are answered with +# `{"entry": }`, `{"skip": True}` or `{"invalid": True}`; `provider` with +# `{"provider": str | None}` or `{"invalid": True}`. +class _BuildStep(_Step): + kind: Literal["validate", "provider", "added"] + path: Literal["create", "add"] + index: int + entry: dict[str, JsonValue] + declared_id: str | None + duplicate_id: bool + provider: str | None + +class _BuiltDeployment(TypedDict): + index: int + id: str | None + provider: str | None + +# The end of a `RouterCoreHandle.build`. +class _BuiltStep(_Step): + kind: Literal["done"] + built: list[_BuiltDeployment] + +# The end of a model-info query: `router_model_info` sets `info` (the merged map or None) and +# `user_keys`; `deployment_model_info` sets `info`; the others set `value`. +class _ModelInfoDoneStep(_Step): + kind: Literal["done"] + info: NotRequired[dict[str, JsonValue] | None] + user_keys: NotRequired[list[str]] + value: NotRequired[JsonValue] + +# `RouterCoreHandle.resolve`: what a name resolves to. +class _Resolution(TypedDict): + model_group: str + kind: str + ids: list[str] + +# The steps each kind of `RouterCall` returns, by the handle method that starts it. +_CallSteps = ( + _AttemptStep + | _SleepStep + | _AllowFallbackStep + | _RefuseFallbacksStep + | _SelectStep + | _PreRouteStep + | _CacheStep + | _DoneStep + | _FailedStep +) +_SelectSteps = _SelectStep | _PreRouteStep | _CacheStep | _SelectedStep | _FailedStep +_HealthySteps = _SelectStep | _PreRouteStep | _CacheStep | _HealthyStep | _FailedStep +_BuildSteps = _SecretsStep | _BuildStep | _BuiltStep | _FailedStep +_ModelInfoSteps = _ModelInfoStep | _ModelInfoDoneStep | _FailedStep +_FailureReportSteps = _CacheStep | _ReportedStep | _FailedStep +_SuccessReportSteps = _CacheStep | _CountedStep | _FailedStep + +class _AttemptSucceeded(TypedDict): + ok: Literal[True] + total_tokens: NotRequired[int] + blocked: NotRequired[Literal["content_filter", "refusal"] | None] + # A stream whose success or failure the host reports when it ends (`report_success`, + # `report_failure`): the attempt counts its request and `total_tokens` only. + deferred: NotRequired[bool] + # Whether the attempt counts against its deployment's usage; unset, its own limits decide. + tracks_usage: NotRequired[bool | None] + +class _AttemptFailed(TypedDict): + ok: Literal[False] + kind: _FailureKind + status: NotRequired[int | None] + retry_after: NotRequired[float | None] + num_retries: NotRequired[int | None] + # The deployment the exception names, which a retry skips over the attempt's own. + failed_deployment_id: NotRequired[str | None] + detail: NotRequired[str | None] + agentic_followup: NotRequired[bool] + # The exception has no `status_code`, which no cooldown rule reads. + status_absent: NotRequired[bool] + # The failure counts against no per-minute requests. + uncounted_request: NotRequired[bool] + +# Each document is a mapping or its JSON text. +_Document = Mapping[str, object] | str + +@final +class RouterCoreHandle: + def __new__(cls, config: _Document, dynamic: _Document) -> RouterCoreHandle: ... + def start(self, op: str, request: _Document) -> RouterCall[_CallSteps]: ... + # Driven like `start`; its `done` step is a `_SelectedStep`. + def select(self, model: str, request: _Document) -> RouterCall[_SelectSteps]: ... + # Driven like `select`; its `done` step is a `_HealthyStep`. + def healthy(self, model: str, request: _Document) -> RouterCall[_HealthySteps]: ... + # `{"replace" | "append" | "upsert": [deployments], "delete": [ids], "settings": {dynamic document}, "index": {...}}`. + def apply(self, changes: _Document) -> int: ... + # Build model_list entries through the host, driven like `start`: `_SecretsStep` and + # `_BuildStep` steps, then a `_BuiltStep`. `path`: "create" (`set_model_list`) or "add"; + # `install`: "replace", "append" or "upsert". + def build( + self, entries: Sequence[object], path: str, resolve_secrets: bool, install: str + ) -> RouterCall[_BuildSteps]: ... + # Keep the providers `other`'s builds resolved for the same deployments. + def adopt_providers(self, other: RouterCoreHandle) -> None: ... + # `get_router_model_info`, driven like `start`: `model_info` lookup steps (a `lookup` field + # and its arguments), then a done step with `info` (the merged map or None) and `user_keys`. + def router_model_info(self, query: _Document) -> RouterCall[_ModelInfoSteps]: ... + # `get_deployment_model_info`, driven like `router_model_info`; its done step has `info`. + def deployment_model_info(self, model_id: str, model_name: object) -> RouterCall[_ModelInfoSteps]: ... + # `_set_model_group_info` for a `GroupInfoQuery` document, driven like `router_model_info`; its + # done step has `value`: None, or `{"assignments": {field: value}}`. + def model_group_info(self, query: _Document) -> RouterCall[_ModelInfoSteps]: ... + # `get_configured_token_limits`, driven like `router_model_info`; `value` is `[input, output]`. + def configured_token_limits(self, id: object, configured: _Document) -> RouterCall[_ModelInfoSteps]: ... + # `get_model_listing_info`, driven like `router_model_info`; `value` has `cost_map_keys`, + # `max_input_tokens` and `max_output_tokens`. + def model_listing_info(self, deployments: list[object]) -> RouterCall[_ModelInfoSteps]: ... + # A configured string that is not blank (`get_configured_mode`, `get_configured_display_name`). + @staticmethod + def configured_text(value: object) -> str | None: ... + # Each model_info's `service_tiers`, unvalidated. + @staticmethod + def configured_service_tiers(model_infos: list[object]) -> list[object]: ... + # The routing-cache key a deployment counts `metric` (tpm, rpm, itpm, otpm) under in `minute`. + def usage_key(self, id: str, model: str, metric: str, minute: str) -> str: ... + def registry_version(self) -> int: ... + # The version the deployments before the newest appends last changed at. + def registry_rewritten(self) -> int: ... + # A failure the router did not see itself, driven like `start`: cache steps, then a + # `_ReportedStep`. + def report_failure( + self, + id: str, + failure: Mapping[str, object], + deployment_cooldown: bool = False, + count_only: bool = False, + requested_group: str | None = None, + cooldown_time: float | None = None, + uncounted: bool = False, + counts_request: bool = False, + usage_model: str | None = None, + callbacks: Mapping[str, bool] | None = None, + ) -> RouterCall[_FailureReportSteps]: ... + # A success a logging callback saw, driven like `report_failure`; `done` carries `key`. + def report_success( + self, + id: str, + total_tokens: float, + requests: float, + usage_model: str | None = None, + tracks_usage: bool | None = None, + callbacks: Mapping[str, bool] | None = None, + ) -> RouterCall[_SuccessReportSteps]: ... + # Count a success against any id, driven like `report_failure`; `done` carries `key`. + def count_success(self, id: str, callbacks: Mapping[str, bool] | None = None) -> RouterCall[_SuccessReportSteps]: ... + # [(id, tpm_key, rpm_key)] for this minute, or None for a group with no deployment. + def usage_keys(self, model_group: str) -> list[tuple[str, str, str]] | None: ... + # Every deployment as a `model_list` entry, in config order. + def deployments(self) -> list[dict[str, JsonValue]]: ... + def deployments_from(self, start: int) -> list[dict[str, JsonValue]]: ... + def deployment(self, id: str) -> dict[str, JsonValue] | None: ... + # Lookups over the registry indexes, answering what each index holds as written. + # Registry lookups (router-core `lookup.rs`): rows the host reads its stored entries at + # (`{"at": position, "name"?, "group"?, "member"?}`, `{"pattern": entry, "name"}`, or a + # `{"raw": value}` / `{"raw_positions": value}` its own read raises at). + def lookup_all( + self, model_name: str, team_id: str | None = None, rename: str | None = None + ) -> list[dict[str, JsonValue]]: ... + def lookup_across_teams(self, model: str) -> tuple[list[dict[str, JsonValue]], int]: ... + def lookup_config(self) -> list[dict[str, JsonValue]]: ... + def lookup_model_ids( + self, model_name: str | None = None, exclude_team_models: bool = False + ) -> tuple[list[str], dict[str, JsonValue] | None]: ... + def lookup_model_list( + self, model_name: str | None, team_id: str | None, scope: Mapping[str, object] + ) -> list[dict[str, JsonValue]]: ... + def lookup_model_names( + self, team_id: str | None, scope: Mapping[str, object] + ) -> tuple[list[str], dict[str, JsonValue] | None]: ... + def lookup_access_groups( + self, model_name: str | None, access_group: str | None, team_id: str | None, scope: Mapping[str, object] + ) -> tuple[list[tuple[str, list[str]]], dict[str, JsonValue] | None]: ... + def lookup_name_serving( + self, model_id: str, scope: Mapping[str, object] + ) -> tuple[bool, str | None, dict[str, JsonValue] | None]: ... + def has_model_name(self, name: str) -> bool: ... + def lookup_team_model_has_alternatives(self, id: str) -> tuple[bool, dict[str, JsonValue] | None]: ... + def lookup_group_usable_by_team( + self, model_group: str, team_id: str | None + ) -> tuple[int | None, dict[str, JsonValue] | None]: ... + def lookup_team_public_name( + self, model_id: str, team_id: str + ) -> tuple[int | None, dict[str, JsonValue] | None]: ... + def lookup_routable(self, model_group: str, team_id: str | None) -> list[dict[str, JsonValue]]: ... + def lookup_wildcard_route( + self, model: str, team_id: str | None, requests: list[str] + ) -> list[dict[str, JsonValue]]: ... + def lookup_candidate_ids( + self, resolved: str, team_id: str | None, members: list[str] | None, requests: list[str] + ) -> tuple[list[str], dict[str, JsonValue] | None]: ... + def has_model_id(self, id: str) -> bool: ... + def id_position(self, id: str) -> JsonValue | None: ... + def id_positions(self) -> list[tuple[str, JsonValue]]: ... + def named_positions(self, name: str) -> JsonValue | None: ... + def team_positions(self, team: str, public: str) -> JsonValue | None: ... + def team_public_names_of(self, team: str) -> list[str]: ... + def litellm_model_positions(self, model: str) -> list[int]: ... + # The registry indexes the host hands out (`HostIndexes`), as JSON. + def host_indexes(self) -> dict[str, JsonValue]: ... + def provider_default_ids(self) -> list[str]: ... + # `PatternMatchRouter.route` over the wildcard router, or `team`'s. + def pattern_route( + self, team: str | None, request: str, filtered: list[str] | None = None + ) -> list[dict[str, JsonValue]] | None: ... + # `get_models_by_pattern` over the wildcard router, or `team`'s. + def models_by_pattern(self, team: str | None, requests: list[str]) -> list[str] | None: ... + # The provider the host resolved when it built deployment `id` (its `provider` build step). + def provider(self, id: str) -> str | None: ... + def resolve(self, name: str, specific_deployment: bool = False) -> _Resolution | None: ... + def routable(self, model: str, request: _Document) -> list[dict[str, JsonValue]]: ... + # `_get_routing_context` for a pick of `model`: (strategy, scope, group). + def routing_context(self, model: str, routing_strategy: str | None = None) -> tuple[str, str, str | None]: ... + def retire(self) -> None: ... + # A stream that failed before content: what may still take over it, and the retry budget for + # a failure of `kind`: {"can_retry", "can_fall_back", "refusal_fallback": bool, + # "retry_budget": int | None, "policy_applies": bool}. + def stream_policy( + self, request: _Document, origin: _Document, kind: str | None = None + ) -> dict[str, bool | int | None]: ... + # The same-group retry and the fallback walk for such a stream, driven like `start`. + def retry_stream( + self, op: str, request: _Document, origin: _Document, failure: Mapping[str, object] + ) -> RouterCall[_CallSteps]: ... + def fallback_stream( + self, op: str, request: _Document, origin: _Document, failure: Mapping[str, object] + ) -> RouterCall[_CallSteps]: ... + def retry_policy_retries(self, failure: Mapping[str, object], model_group: str | None = None) -> int | None: ... + def allowed_fails_for(self, failure: Mapping[str, object]) -> int | None: ... + def should_retry( + self, failure: Mapping[str, object], counts: tuple[int, int], fallbacks: tuple[bool, bool, bool] + ) -> bool: ... + +_S = TypeVar("_S", covariant=True) + +@final +class RouterCall(Generic[_S]): + # The answer to the last step, which Rust validates against it: attempt: _AttemptSucceeded | + # _AttemptFailed; sleep: None; allow_fallback: bool; refuse_fallbacks: bool; select: the remaining + # candidate ids, pick first; pre_route: see _PreRouteStep; cache: see _CacheStep; secrets, build and model_info: + # see those steps; first call: None. + def next(self, outcome: object) -> _S: ... diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 9ba8cbc35a7..a12f79c6c8a 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -23,6 +23,7 @@ class Route(str, Enum): OCR = "ocr" TOKEN_COUNTER = "token_counter" TOKENIZER = "tokenizer" + ROUTER = "router" class Target(str, Enum): @@ -224,6 +225,7 @@ RULES: Final[Rules] = ( RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY), + RouteRule(Route.ROUTER, Rollout.RUST_OPT_IN), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), ) diff --git a/litellm/types/integrations/model_deprecation_email.py b/litellm/types/integrations/model_deprecation_email.py new file mode 100644 index 00000000000..f196bb47544 --- /dev/null +++ b/litellm/types/integrations/model_deprecation_email.py @@ -0,0 +1,35 @@ +"""Model retirement email milestones and recipient-scoped content.""" + +from dataclasses import dataclass +from datetime import date +from enum import IntEnum +from typing import Final + +from litellm.types.proxy.model_deprecation import ModelDeprecationInfo + +MODEL_DEPRECATION_EMAIL_EVENT: Final = "Model Deprecations" + + +class DeprecationEmailMilestone(IntEnum): + THIRTY_DAYS = 30 + ONE_DAY = 1 + + +@dataclass(frozen=True, slots=True) +class ModelDeprecationEmail: + model: ModelDeprecationInfo + usage_model: str + recipient_email: str + milestone: DeprecationEmailMilestone + key_count: int + requests: int + team_names: tuple[str, ...] + start_date: date + end_date: date + + +@dataclass(frozen=True, slots=True) +class RenderedDeprecationEmail: + subject: str + html: str + text: str diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py index 49dd8b9ef91..5d8479b8ce1 100644 --- a/litellm/types/proxy/claude_code_endpoints.py +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -2,6 +2,7 @@ Claude Code Marketplace endpoint types for LiteLLM Proxy """ +from datetime import datetime from typing import Final, Literal, TypeAlias from pydantic import AliasChoices, ConfigDict, Field, TypeAdapter, model_validator @@ -32,7 +33,7 @@ class PluginMetadata(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - version: str | None = Field("1.0.0", description="Semantic version") + version: str | None = Field("1.0.0", description="Semantic version, defaults to 1.0.0 on creation") description: str | None = Field(None, description="Plugin description") author: PluginAuthor | None = Field(None, description="Plugin author") homepage: str | None = Field(None, description="Plugin homepage URL") @@ -105,11 +106,14 @@ class UpdatePluginRequest(PluginSpec): The plugin name is the resource identity and is supplied as the path parameter, so it cannot be changed here. This is a full replace: omitted - fields reset to their defaults, so version is cleared rather than - defaulting to the create-time "1.0.0". + fields reset to their defaults, except version which automatically increments + the published patch version when omitted or null. """ - version: str | None = Field(None, description="Semantic version; cleared if omitted") + version: str | None = Field(None, description="Explicit higher SemVer, or omit/null for an automatic patch bump") + expected_updated_at: datetime | None = Field( + None, description="Published updated_at read by the editor. A stale value returns 409", exclude=True + ) class PluginResponse(LiteLLMBaseModel): @@ -150,6 +154,7 @@ class PluginListItem(LiteLLMBaseModel): enabled: bool created_at: str | None updated_at: str | None + has_agent_skill_metadata: bool = Field(default=False, exclude=True) class ListPluginsResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 3699f01a684..37c49d737af 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -1,9 +1,10 @@ from __future__ import annotations from datetime import date, datetime +from enum import Enum from typing import Final, Literal -from pydantic import Field +from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel @@ -16,8 +17,17 @@ DEPRECATION_IDLE_POLL_SECONDS: Final = 30 DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] +class SuccessorModelSource(str, Enum): + MODEL_COST_MAP = "model_cost_map" + DEPLOYMENT_CONFIG = "deployment_config" + + class ModelDeprecationInfo(LiteLLMBaseModel): model_name: str = Field(description="The public name of the model on the proxy (model_group).") + team_public_model_name: str | None = Field( + default=None, + description="The model name visible to the team, when configured.", + ) litellm_model: str | None = Field( default=None, description="The underlying litellm model string the deprecation date is sourced from.", @@ -40,6 +50,14 @@ class ModelDeprecationInfo(LiteLLMBaseModel): ), ) + successor_model_source: SuccessorModelSource | None = Field( + default=None, + description="Whether the recommended successor comes from the model cost map or deployment configuration.", + ) + deprecation_source: str | None = Field( + default=None, description="Provider deprecation notice linked by the selected model cost-map entry." + ) + class ModelDeprecationResponse(LiteLLMBaseModel): deprecated: list[ModelDeprecationInfo] = Field( @@ -59,3 +77,38 @@ class ModelDeprecationResponse(LiteLLMBaseModel): ) warn_within_days: int = Field(description="The window (in days) used to bucket 'imminent' models.") checked_at: datetime = Field(description="UTC timestamp when the deprecation snapshot was generated.") + + +class DeprecationAlertStatus(str, Enum): + OPEN = "open" + RESOLVED = "resolved" + DISMISSED = "dismissed" + + +class DeprecationAlertUpdate(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str = Field(min_length=1, max_length=1024) + deprecation_date: date + status: DeprecationAlertStatus + last_seen_date: date + last_seen_requests: int = Field(ge=0) + + +class DeprecationAlertState(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + deprecation_date: date + status: DeprecationAlertStatus = DeprecationAlertStatus.OPEN + last_seen_date: date | None = None + last_seen_requests: int = 0 + updated_at: datetime | None = None + updated_by: str | None = None + + +class DeprecationTagUsage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + api_key: str + tag: str diff --git a/litellm/types/proxy/skill_hub.py b/litellm/types/proxy/skill_hub.py new file mode 100644 index 00000000000..d575defcda7 --- /dev/null +++ b/litellm/types/proxy/skill_hub.py @@ -0,0 +1,165 @@ +from collections.abc import Mapping +from datetime import datetime +from typing import Literal, TypeAlias + +from pydantic import ConfigDict, Field + +from litellm.models.skill_submission import SubmissionOperation, SubmissionStatus +from litellm.types.llms.base import LiteLLMBaseModel + + +class SkillHubSkill(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + id: str + name: str + description: str | None = None + version: str | None = None + updated_at: datetime | None = None + visibility: Literal["public", "private"] + source: Mapping[str, str] + download_url: str | None = None + content_url: str | None = None + domain: str | None = None + namespace: str | None = None + keywords: tuple[str, ...] | None = None + download_count: int | None = Field( + None, description="Gateway ZIP downloads, included in proxy-admin catalog listings" + ) + + +class SkillHubList(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + skills: tuple[SkillHubSkill, ...] + count: int + next_offset: int | None + + +class ImportSkillRequest(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + url: str = Field(description="Public skills.sh skill URL or GitHub repository URL") + skill: str | None = Field(None, description="Skill name to select when a repository contains several skills") + ref: str = Field("HEAD", description="Git commit, tag or branch to import") + path: str | None = Field(None, description="Repository-relative skill directory") + version: str | None = Field(None, description="Semantic version, or omit/null to start at 1.0.0") + + +class SkillHubMetadata(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + description: str | None = Field(None, max_length=1024) + version: str | None = Field(None, description="Explicit higher SemVer, or omit/null for an automatic patch bump") + expected_updated_at: datetime | None = Field( + None, description="Published updated_at read by the editor. A stale value returns 409", exclude=True + ) + domain: str | None = None + namespace: str | None = None + keywords: tuple[str, ...] | None = None + + +class SkillHubPublication(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class SkillHubVersion(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + id: str + version: str | None + description: str | None + manifest: Mapping[str, object] + published_at: datetime | None + published_by: str | None + content_kind: Literal["stored_archive", "source_reference"] + download_url: str | None + + +class SkillHubVersions(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + versions: tuple[SkillHubVersion, ...] + next_offset: int | None + + +class SkillHubAgentSkill(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + name: str + description: str + digest: str + origin: str | None + + +class SkillHubProposal(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + name: str + description: str | None + version: str | None + domain: str | None = None + namespace: str | None = None + keywords: tuple[str, ...] | None = None + source: Mapping[str, str] + agent_skill: SkillHubAgentSkill | None = None + + +class SkillHubPendingSkill(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["pending_review"] = "pending_review" + action: Literal["submitted"] = "submitted" + submission_id: str + proposal: SkillHubProposal + + +SkillHubMutationResponse: TypeAlias = SkillHubSkill | SkillHubPendingSkill + + +class SkillHubSubmission(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + id: str + skill_name: str + operation: SubmissionOperation + status: SubmissionStatus + proposal: SkillHubProposal + has_archive: bool + submitted_by: str + created_at: datetime + base_skill_id: str | None + base_updated_at: datetime | None + reviewed_by: str | None + reviewed_at: datetime | None + rejection_reason: str | None + + +class SkillHubSubmissionList(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + submissions: tuple[SkillHubSubmission, ...] + count: int + has_more: bool + + +class SkillHubSubmissionDetail(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + submission: SkillHubSubmission + current_skill: SkillHubSkill | None + + +class SkillHubReviewResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["success"] = "success" + action: Literal["approved", "rejected"] + submission: SkillHubSubmission + skill: SkillHubSkill | None + + +class SkillHubRejection(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + reason: str | None = Field(None, max_length=2000) diff --git a/litellm/types/proxy/skill_scanning.py b/litellm/types/proxy/skill_scanning.py new file mode 100644 index 00000000000..19577c8a9ac --- /dev/null +++ b/litellm/types/proxy/skill_scanning.py @@ -0,0 +1,91 @@ +from datetime import datetime +from typing import Annotated, Final, Literal +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class SkillScannerConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + endpoint: str = Field(max_length=2048, description="External scanner's authenticated POST endpoint") + api_key_env: str = Field( + pattern=r"^[A-Za-z_][A-Za-z0-9_]*$", description="Environment secret name for scanner auth" + ) + use_llm: bool = False + timeout_seconds: int = Field(default=180, ge=10, le=600) + + @field_validator("endpoint") + @classmethod + def validate_endpoint(cls, value: str) -> str: + parsed: Final = urlsplit(value) + if ( + not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or not ( + parsed.scheme == "https" + or (parsed.scheme == "http" and parsed.hostname in {"127.0.0.1", "::1", "localhost"}) + ) + ): + raise ValueError( + "Scanner endpoint must use HTTPS without credentials, query or fragment (HTTP loopback allowed)" + ) + return value + + +class SkillScanFinding(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + rule_id: str = Field(max_length=200) + severity: Literal["LOW", "MEDIUM", "HIGH", "CRITICAL"] + message: str = Field(max_length=8000) + file: str = Field(max_length=2048) + start_line: int | None = Field(default=None, ge=1) + explanation: str | None = Field(default=None, max_length=8000) + remediation: str | None = Field(default=None, max_length=8000) + + +class SkillScanReport(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + scanner_version: str = Field(min_length=1, max_length=100) + complete: bool + scan_mode: Literal["static-only", "static+llm"] + risk_score: int = Field(ge=0, le=100) + findings: tuple[SkillScanFinding, ...] = Field(max_length=500) + limitations: tuple[Annotated[str, Field(max_length=2000)], ...] = Field(default=(), max_length=100) + + +class SkillScan(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + status: Literal["running", "completed", "failed"] + requested_by: str + started_at: datetime + expires_at: datetime + completed_at: datetime | None = None + archive_sha256: str | None = None + report: SkillScanReport | None = None + error: str | None = None + + def at(self, now: datetime) -> "SkillScan": + if self.status == "running" and self.expires_at <= now: + return self.model_copy( + update={"status": "failed", "error": "Scan timed out or was interrupted. Try scanning again."} + ) + return self + + +class SkillScanRequest(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class SkillScanResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + configured: bool + scan: SkillScan | None diff --git a/litellm/types/router.py b/litellm/types/router.py index 4e3e4d92b3c..3363d113fc7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1154,9 +1154,9 @@ class CustomRoutingStrategyBase: self, model: str, messages: list[dict[str, str]] | None = None, - input: str | list | None = None, + input: str | list[object] | None = None, specific_deployment: bool | None = False, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, ) -> None: """ Asynchronously retrieves the available deployment based on the given parameters. @@ -1177,9 +1177,9 @@ class CustomRoutingStrategyBase: self, model: str, messages: list[dict[str, str]] | None = None, - input: str | list | None = None, + input: str | list[object] | None = None, specific_deployment: bool | None = False, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, ) -> None: """ Synchronously retrieves the available deployment based on the given parameters. diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4c47489e08f..ba29a67a36c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -224,6 +224,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_adaptive_thinking: bool | None supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] + thinking_on_by_default: ReadOnly[bool | None] supports_tool_search: bool | None supports_mid_conversation_system: bool | None supports_url_context: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index 4add9d53cb3..53b02ebddac 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4305,6 +4305,13 @@ class _LiveDeploymentReplay: """ callback: Callable[[], None] | None = None + # Replays other router cores add beside the slot, run after it. + extra: tuple[Callable[[], None], ...] = () + + +def add_live_deployment_replay(replay: Callable[[], None]) -> None: + """Add a callback that re-asserts another router core's live deployments after a refresh.""" + _LiveDeploymentReplay.extra = (*_LiveDeploymentReplay.extra, replay) def set_live_deployment_replay(replay: Callable[[], None]) -> None: @@ -4338,6 +4345,8 @@ def reapply_runtime_model_cost_registrations() -> None: """ if _LiveDeploymentReplay.callback is not None: _LiveDeploymentReplay.callback() + for extra_replay in _LiveDeploymentReplay.extra: + extra_replay() if _runtime_registered_model_cost: register_model( model_cost=dict( @@ -6338,7 +6347,15 @@ def get_deployment_order(deployment: Mapping[str, object]) -> int | None: resolved_order: Final = ( order if order is not None else cast(Mapping[str, object], deployment.get("model_info", {})).get("order") ) - return cast(int | None, resolved_order) + # One number for every spelling (`1`, `1.0`, `"1"`, `True`), so orders written different ways + # sort and compare together; anything else counts as no order. + if isinstance(resolved_order, bool | int): + return int(resolved_order) + if isinstance(resolved_order, float): + return int(resolved_order) if resolved_order.is_integer() else None + if isinstance(resolved_order, str) and re.fullmatch(r"[+-]?[0-9]+", resolved_order): + return int(resolved_order) + return None _get_deployment_order = get_deployment_order @@ -7581,6 +7598,7 @@ def get_model_info_helper( supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), + thinking_on_by_default=_model_info.get("thinking_on_by_default", None), supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 25c7c3593ed..76fd85d001b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2125,6 +2125,7 @@ "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2213,6 +2214,7 @@ "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2301,6 +2303,7 @@ "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2389,6 +2392,7 @@ "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2472,6 +2476,7 @@ "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2560,6 +2565,7 @@ "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2958,6 +2964,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3002,6 +3009,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3046,6 +3054,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3090,6 +3099,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3134,6 +3144,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3178,6 +3189,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -4063,6 +4075,7 @@ "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -4204,6 +4217,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -16507,6 +16521,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -17042,6 +17057,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -21910,6 +21926,7 @@ "prompt_cache_min_tokens": 512, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -22068,6 +22085,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -36308,6 +36326,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -36331,6 +36350,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -36350,6 +36370,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -36385,6 +36406,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -36420,6 +36442,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -39774,6 +39797,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 8e-06, + "output_cost_per_image_token_batches": 4e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -50486,6 +50510,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -50563,6 +50588,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -52968,6 +52994,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53006,6 +53033,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53255,6 +53283,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -59906,6 +59935,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -60629,6 +60659,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -64719,7 +64750,8 @@ { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", - "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number. On mapped Claude entries from Anthropic, Azure Foundry, Bedrock and Vertex it only fills the keys an entry leaves out, so the entry's own values always win.", "model_info": { "mode": "chat", "max_input_tokens": 200000, @@ -64762,6 +64794,14 @@ "thinking_always_on": true } }, + { + "name": "claude-thinking-on-by-default", + "pattern": "claude-[a-z]+-[5-9](?!\\d)", + "description": "Claude at major 5 or higher, in any id shape that contains claude--. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.", + "model_info": { + "thinking_on_by_default": true + } + }, { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", @@ -69285,6 +69325,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69360,6 +69401,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69616,6 +69658,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69681,6 +69724,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -75432,6 +75476,7 @@ "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -75548,6 +75593,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77052,6 +77098,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77104,6 +77151,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77148,6 +77196,7 @@ "thinking_always_on": false, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77195,6 +77244,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77231,6 +77281,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77287,6 +77338,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77333,6 +77385,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77373,6 +77426,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77414,6 +77468,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77460,6 +77515,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77511,6 +77567,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77557,6 +77614,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77617,6 +77675,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77668,6 +77727,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77718,6 +77778,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77762,6 +77823,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -79521,6 +79583,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79735,6 +79798,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79901,6 +79965,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -81886,6 +81951,7 @@ "prompt_cache_min_tokens": 512.0, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, "supports_function_calling": true, @@ -85880,5 +85946,235 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "gpt-image-2.6-sunburst": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "gpt-image-2.6-flare": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-west-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock_mantle/xai.grok-4.7": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_flex": 2.75e-07, + "cache_read_input_token_cost_priority": 9.625e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "output_cost_per_token_batches": 3.3e-06, + "output_cost_per_token_flex": 3.3e-06, + "output_cost_per_token_priority": 1.155e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "together_ai/ricursive-ai/Ripple-65B-Preview": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.together.ai/v1/models", + "supports_prompt_caching": true } } diff --git a/pyproject.toml b/pyproject.toml index 73cdb1fc46a..eb08a24490f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ proxy = [ "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "expression>=5.6.0,<6.0", "backports-zstd>=1.0.0,<2.0; python_version < '3.14'", + "semver>=3.0.4,<4", ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy # imports are all guarded, so it runs on the base SDK plus these packages, and @@ -156,7 +157,8 @@ stt-vertex-chirp = [ stt-nvidia-riva = [ # NVIDIA Riva STT provider (gRPC). These are imported lazily inside the # provider handler so litellm core remains usable without them. - "nvidia-riva-client>=2.15.0", + "grpcio>=1.67.1,<2.0", + "protobuf>=5.26.1,<7.0", "soundfile>=0.12.1", "audioread>=3.0.1", "numpy>=1.26.0", @@ -242,6 +244,7 @@ dev = [ "pytest-timeout==2.4.0", "vcrpy==8.2.1", "pytest-recording==0.13.4", + "nvidia-riva-client==2.16.0", ] e2e-dev = [ "playwright==1.61.0", @@ -268,6 +271,7 @@ proxy-dev = [ "a2a-sdk==1.1.0", "backports-zstd==1.8.0; python_version < '3.14'", "httptools>=0.7.1,<0.8", + "semver>=3.0.4,<4", ] ci = [ "psutil==7.2.2", @@ -276,6 +280,9 @@ ci = [ # protobuf, Pillow is a compiled C extension). "tenacity==8.5.0", "google-generativeai==0.8.6", + # The Riva tests serve NVIDIA's own message definitions, which checks + # LiteLLM's hand-written copy of them on every run. + "nvidia-riva-client==2.16.0", "Pillow==12.3.0; python_version == '3.10'", "Pillow==13.0.0; python_version >= '3.11'", # Azure batch E2E tests still import psycopg2 directly. diff --git a/schema.prisma b/schema.prisma index c007b047087..818f2dc4858 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1616,10 +1616,69 @@ model LiteLLM_ClaudeCodePluginTable { created_by String? uploaded_archive LiteLLM_ClaudeCodePluginUpload? + versions LiteLLM_ClaudeCodePluginVersion[] + downloads LiteLLM_ClaudeCodePluginDownload[] + anonymous_download_count BigInt @default(0) @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginVersion { + id String @id @default(uuid()) + plugin_id String + version String? + description String? + manifest_json String? + published_at DateTime? + published_by String? + archive_available Boolean @default(false) + content Bytes? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@unique([plugin_id, version]) + @@index([plugin_id, published_at]) +} + + +model LiteLLM_SkillSubmission { + id String @id @default(uuid()) + plugin_name String + operation String + status String @default("pending_review") + proposal_json String + archive_content Bytes? + submitted_by String + created_at DateTime @default(now()) + base_plugin_id String? + base_updated_at DateTime? + reviewed_by String? + reviewed_at DateTime? + rejection_reason String? + scan_json String? + + @@index([submitted_by, created_at(sort: Desc), id(sort: Desc)]) + @@index([submitted_by, status, created_at(sort: Desc), id(sort: Desc)]) + @@index([status, created_at(sort: Desc), id(sort: Desc)]) + @@index([created_at(sort: Desc), id(sort: Desc)]) +} + +model LiteLLM_ClaudeCodePluginDownload { + id BigInt @id @default(autoincrement()) + plugin_id String + downloaded_at DateTime @default(now()) + version String? + digest String + authenticated Boolean + user_id String? + user_email String? + key_alias String? + team_id String? + team_alias String? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@index([plugin_id, id]) +} + model LiteLLM_ClaudeCodePluginUpload { plugin_name String @id content Bytes @@ -2071,74 +2130,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/scripts/check_schema_compatibility.py b/scripts/check_schema_compatibility.py index b8e9ac3c07a..c56986a38c8 100644 --- a/scripts/check_schema_compatibility.py +++ b/scripts/check_schema_compatibility.py @@ -13,6 +13,7 @@ from pydantic import TypeAdapter Schema = Mapping[str, Mapping[str, str]] MODELS: Final = re.compile(r"^[ \t]*model\s+(\w+)\s*\{(.*?)^[ \t]*\}", re.MULTILINE | re.DOTALL) +DROPPED_UNUSED_MODELS: Final = frozenset({"LiteLLM_AccountingDelivery", "LiteLLM_AccountingEvent"}) def parse_model_fields(schema: str) -> Schema: @@ -27,18 +28,18 @@ def _field_changes(model: str, field: str, old_type: str, new_fields: Mapping[st if field not in new_fields: yield f"Breaking: Field {model}.{field} was removed" return - new_type: Final = new_fields[field] - if old_type == new_type: - return - if "?" in old_type and "?" not in new_type: + old_base: Final = old_type.split()[0] + new_base: Final = new_fields[field].split()[0] + if old_base.endswith("?") and not new_base.endswith("?"): yield f"Breaking: Field {model}.{field} changed from optional to required" - if not old_type.startswith(new_type.split("?")[0]): - yield f"Breaking: Field {model}.{field} changed type from {old_type} to {new_type}" + if old_base.rstrip("?") != new_base.rstrip("?"): + yield f"Breaking: Field {model}.{field} changed type from {old_base} to {new_base}" def _model_changes(model: str, old_fields: Mapping[str, str], new_schema: Schema) -> Iterator[str]: if model not in new_schema: - yield f"Breaking: Model {model} was removed" + if model not in DROPPED_UNUSED_MODELS: + yield f"Breaking: Model {model} was removed" return for field, old_type in old_fields.items(): yield from _field_changes(model, field, old_type, new_schema[model]) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a84a40c67c1..f4ca7fa2965 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -87,6 +87,10 @@ IGNORE_FUNCTIONS = [ "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. "_merge_session_fields", # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update. + "_json", # router_rust index views: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap. + "to_wire", # router_rust model info: bounded by the nesting depth of a stored deployment or cost-map entry (finite JSON trees: pydantic refuses a cycle when it dumps a deployment). + "from_wire", # router_rust model info: walks only what router-core merged from to_wire's output, so it never goes deeper than to_wire did. + "_valid_frontmatter_node", # max depth 32, manifest size capped at 1 MiB, and YAML aliases rejected before traversal. ] diff --git a/tests/code_coverage_tests/router_litellm_globals_check.py b/tests/code_coverage_tests/router_litellm_globals_check.py index ff9fac0e3dc..172a3754000 100644 --- a/tests/code_coverage_tests/router_litellm_globals_check.py +++ b/tests/code_coverage_tests/router_litellm_globals_check.py @@ -21,6 +21,7 @@ CHECKED = ( "litellm/router_core.py", "litellm/router_deprecated.py", "litellm/router_python_core.py", + "litellm/router_rust/core.py", ) diff --git a/tests/code_coverage_tests/router_surface_check.py b/tests/code_coverage_tests/router_surface_check.py index a8e0d6fa0d7..1c41526101a 100644 --- a/tests/code_coverage_tests/router_surface_check.py +++ b/tests/code_coverage_tests/router_surface_check.py @@ -11,7 +11,8 @@ modules listed there. The lists hold what is used today and may only shrink: also fails, so the entry is removed in the same change. "Outside" means every non-test file under litellm/ and enterprise/ except -litellm/router.py and the router's own modules. Uses from the router family +litellm/router.py and the router's own modules (including every +module under litellm/router_rust/, the Rust core's host-side glue). Uses from the router family (litellm/router_utils, litellm/router_strategy) are listed separately, because those modules become private parts of the router core in a later step. @@ -42,9 +43,12 @@ ROUTER_FILES = { "litellm/router_core.py", "litellm/router_python_core.py", "litellm/router_python_host.py", + "litellm/router_rust/core.py", "litellm/router_deprecated.py", "litellm/router_host.py", } +# The Rust core's host-side glue modules are part of the router. +ROUTER_FILE_PREFIXES = ("litellm/router_rust/",) ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_python_core.py", "litellm/router_deprecated.py"] ROUTER_CLASS_NAMES = {"Router", "PythonCore", "DeprecatedRouterMembers"} @@ -128,7 +132,7 @@ def scan() -> Dict[str, Dict[str, Set[str]]]: uses: Dict[str, Dict[str, Set[str]]] = {"outside": {}, "router_family": {}} imports: Dict[str, Set[str]] = {} for rel, path in _python_files(): - if rel in ROUTER_FILES or _is_test(rel): + if rel in ROUTER_FILES or rel.startswith(ROUTER_FILE_PREFIXES) or _is_test(rel): continue with open(path, encoding="utf-8") as f: src = f.read() diff --git a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts index f8ce4faf45e..517ec64c7df 100644 --- a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts +++ b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts @@ -1,8 +1,8 @@ -import { test, expect, type APIRequestContext } from "@playwright/test"; +import { test, expect } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; -import { masterKey } from "../../helpers/traffic"; +import { masterKey, uniqueSuffix } from "../../helpers/traffic"; test.describe("AI Hub (internal admin view)", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -62,103 +62,93 @@ test.describe("AI Hub (internal admin view)", () => { }); test.describe("Public model hub (/ui/model_hub_table)", () => { - // No storageState — the public page is reached anonymously with a `key` query param. - - test("Public model_hub_table loads and renders the Models tab", async ({ page }) => { - // The page expects the proxy key as the `key` query param. Use the master - // key the e2e runner already exports — this matches what the AI Hub copy - // button hands out. - await page.goto(`/ui/model_hub_table?key=${masterKey()}`); - - // Dismiss the feedback popup before asserting on the tab, so a popup - // race can't briefly mask the tab while we're evaluating visibility. + test("shows the Model Hub and Skills Hub catalogs", async ({ page }) => { + await page.goto("/ui/model_hub_table"); await dismissFeedbackPopup(page); - await expect(page.getByRole("tab", { name: /^Models/ })).toBeVisible({ timeout: 10_000 }); + await expect( + page.getByRole("tab", { name: "Model Hub", exact: true }), + ).toBeVisible(); + await expect( + page.getByRole("textbox", { name: "Search models", exact: true }), + ).toBeVisible(); + await expect(page.getByRole("tab")).toHaveCount(2); + await page.getByRole("tab", { name: "Skills Hub", exact: true }).click(); + await expect( + page.getByRole("textbox", { name: "Search skills", exact: true }), + ).toBeVisible(); }); - test("Agents and MCP servers tabs render their public entries", async ({ page, request }) => { - const suffix = `${Date.now()}`; - const agentName = `e2e-public-agent-${suffix}`; - const mcpServerName = `e2e_public_mcp_${suffix}`; + test("finds a published skill and opens its installation instructions", async ({ + page, + request, + }) => { + const skillName = `e2e-catalog-${uniqueSuffix()}`; + const domain = `catalog-${uniqueSuffix()}`; const auth = { Authorization: `Bearer ${masterKey()}` }; - - const publicMcpServerIds = async (api: APIRequestContext): Promise => { - const res = await api.get("/public/mcp_hub"); - expect(res.ok(), `public mcp_hub read failed (${res.status()}): ${await res.text()}`).toBe(true); - const servers: { server_id: string }[] = await res.json(); - return servers.map((server) => server.server_id); - }; - - const seedPublicEntries = async ( - api: APIRequestContext, - priorMcpIds: string[], - ): Promise<{ agentId: string; serverId: string }> => { - const agentRes = await api.post("/v1/agents", { - headers: auth, - data: { - agent_name: agentName, - agent_card_params: { - name: agentName, - description: "E2E public agent", - version: "1.0.0", - url: "http://127.0.0.1:9999/", - capabilities: {}, - skills: [], - defaultInputModes: ["text"], - defaultOutputModes: ["text"], - }, - }, - }); - expect(agentRes.ok(), `agent create failed (${agentRes.status()}): ${await agentRes.text()}`).toBe(true); - const agentId = (await agentRes.json()).agent_id as string; - - const serverRes = await api.post("/v1/mcp/server", { - headers: auth, - data: { - server_name: mcpServerName, - url: "http://127.0.0.1:9999/mcp", - transport: "http", - description: "E2E public MCP server", - }, - }); - expect(serverRes.ok(), `mcp server create failed (${serverRes.status()}): ${await serverRes.text()}`).toBe(true); - const serverId = (await serverRes.json()).server_id as string; - - const agentPublicRes = await api.post(`/v1/agents/${agentId}/make_public`, { headers: auth }); - expect(agentPublicRes.ok(), `agent make_public failed: ${await agentPublicRes.text()}`).toBe(true); - const mcpPublicRes = await api.post("/v1/mcp/make_public", { - headers: auth, - data: { mcp_server_ids: [...priorMcpIds, serverId] }, - }); - expect(mcpPublicRes.ok(), `mcp make_public failed: ${await mcpPublicRes.text()}`).toBe(true); - - return { agentId, serverId }; - }; - - const priorMcpIds = await publicMcpServerIds(request); - const { agentId, serverId } = await seedPublicEntries(request, priorMcpIds); + const created = await request.post("/claude-code/plugins", { + headers: auth, + data: { + name: skillName, + source: { source: "github", repo: "example/catalog-skills" }, + description: "Catalog browser test skill", + domain, + version: "1.0.0", + }, + }); + expect( + created.ok(), + `skill registration failed (${created.status()}): ${await created.text()}`, + ).toBe(true); try { - await page.goto(`/ui/model_hub_table?key=${masterKey()}`); + await page.goto("/ui/model_hub_table?tab=skills"); await dismissFeedbackPopup(page); + await page.getByRole("checkbox", { name: domain, exact: true }).check(); + await page + .getByRole("textbox", { name: "Search skills", exact: true }) + .fill(skillName); + await page.getByRole("link").filter({ hasText: skillName }).click(); - const agentsTab = page.getByRole("tab", { name: /^Agents/ }); - await expect(agentsTab).toBeVisible({ timeout: 15_000 }); - await agentsTab.click(); - await expect(page.getByRole("textbox", { name: "Search agents" })).toBeVisible({ timeout: 10_000 }); - await expect(page.getByRole("link").filter({ hasText: agentName })).toHaveCount(1, { timeout: 10_000 }); - await expect(page.getByText("E2E public agent").first()).toBeVisible(); - - const mcpTab = page.getByRole("tab", { name: /^MCP/ }); - await expect(mcpTab).toBeVisible(); - await mcpTab.click(); - await expect(page.getByRole("textbox", { name: "Search MCP servers" })).toBeVisible({ timeout: 10_000 }); - await expect(page.getByRole("link").filter({ hasText: mcpServerName })).toHaveCount(1, { timeout: 10_000 }); - await expect(page.getByText("E2E public MCP server").first()).toBeVisible(); + const details = page.getByRole("dialog"); + await expect( + details.getByRole("heading", { name: skillName, exact: true }), + ).toBeVisible(); + await expect( + details.getByText(`/plugin install ${skillName}@litellm`, { + exact: true, + }), + ).toBeVisible(); + await expect(page).toHaveURL(new RegExp(`open=${skillName}`)); + await page.reload(); + await expect( + page + .getByRole("dialog") + .getByRole("heading", { name: skillName, exact: true }), + ).toBeVisible(); } finally { - await request.post("/v1/mcp/make_public", { headers: auth, data: { mcp_server_ids: priorMcpIds } }); - await request.delete(`/v1/agents/${agentId}`, { headers: auth }); - await request.delete(`/v1/mcp/server/${serverId}`, { headers: auth }); + const deleted = await request.delete( + `/claude-code/plugins/${skillName}`, + { headers: auth }, + ); + expect(deleted.ok(), `skill cleanup failed (${deleted.status()})`).toBe( + true, + ); + } + }); + + test("retired catalog links return to Models and clear the old selection", async ({ + page, + }) => { + for (const tab of ["agents", "mcp"]) { + await page.goto( + `/ui/model_hub_table?tab=${tab}&open=retired-entry&key=preserved`, + ); + await dismissFeedbackPopup(page); + await expect( + page.getByRole("tab", { name: "Model Hub", exact: true }), + ).toHaveAttribute("aria-selected", "true"); + await expect(page).toHaveURL(/model_hub_table\?key=preserved$/); + await expect(page.getByRole("dialog")).toHaveCount(0); } }); }); diff --git a/tests/integration/_support/riva.py b/tests/integration/_support/riva.py index 8552bc2197d..438d877386b 100644 --- a/tests/integration/_support/riva.py +++ b/tests/integration/_support/riva.py @@ -39,6 +39,7 @@ class RivaCall: rpc: Rpc audio: bytes cancelled: bool + authorization: tuple[str, ...] = () @dataclass(frozen=True, slots=True) @@ -68,6 +69,10 @@ def pcm_bytes(seconds: float) -> int: return int(seconds * SAMPLE_RATE_HZ) * 2 +def _authorization(context: grpc.ServicerContext) -> tuple[str, ...]: + return tuple(str(value) for key, value in context.invocation_metadata() if key == "authorization") + + def carries_clip(audio: bytes, seconds: float) -> bool: sent: Final = _clip_samples(seconds) if len(audio) != len(sent) * 2: @@ -100,9 +105,9 @@ class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer): released: Final = threading.Event() context.add_callback(released.set) released.wait(timeout=_STALL_LIMIT_SECONDS) - self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, cancelled=not context.is_active())) + self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, not context.is_active(), _authorization(context))) return riva_asr_pb2.RecognizeResponse() - self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, cancelled=False)) + self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, False, _authorization(context))) return riva_asr_pb2.RecognizeResponse( results=[riva_asr_pb2.SpeechRecognitionResult(alternatives=[self._alternative(request.config)])] ) @@ -111,13 +116,12 @@ class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer): self, request_iterator: Iterable[riva_asr_pb2.StreamingRecognizeRequest], context: grpc.ServicerContext ) -> Iterator[riva_asr_pb2.StreamingRecognizeResponse]: if self._offline_only: - self._calls.put(RivaCall(Rpc.STREAMING, b"", cancelled=False)) + self._calls.put(RivaCall(Rpc.STREAMING, b"", False, _authorization(context))) context.abort(grpc.StatusCode.INVALID_ARGUMENT, OFFLINE_ONLY_REFUSAL) requests: Final = tuple(request_iterator) config: Final = requests[0].streaming_config.config if requests else riva_asr_pb2.RecognitionConfig() - self._calls.put( - RivaCall(Rpc.STREAMING, b"".join(request.audio_content for request in requests), cancelled=False) - ) + audio: Final = b"".join(request.audio_content for request in requests) + self._calls.put(RivaCall(Rpc.STREAMING, audio, False, _authorization(context))) yield riva_asr_pb2.StreamingRecognizeResponse( results=[riva_asr_pb2.StreamingRecognitionResult(alternatives=[self._alternative(config)], is_final=True)] ) diff --git a/tests/integration/database/migrations/test_drop_accounting_event_tables.py b/tests/integration/database/migrations/test_drop_accounting_event_tables.py new file mode 100644 index 00000000000..583223a573b --- /dev/null +++ b/tests/integration/database/migrations/test_drop_accounting_event_tables.py @@ -0,0 +1,64 @@ +import os +from pathlib import Path +from typing import Final + +import psycopg +import pytest + +_MIGRATIONS: Final = Path(__file__).parents[4] / "litellm-proxy-extras/litellm_proxy_extras/migrations" +_OLD_LEDGER: Final = _MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql" +_SPEND_LOG_DELIVERY: Final = _MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql" +_DROP: Final = _MIGRATIONS / "20261104010000_drop_accounting_event_tables/migration.sql" +_DROPPED: Final = ( + "AccountingDeliveryTarget", + "AccountingEventKind", + "LiteLLM_AccountingDelivery", + "LiteLLM_AccountingEvent", +) + + +def _apply(connection: psycopg.Connection[tuple[object, ...]], migration: Path) -> None: + with connection.transaction(): + connection.execute(migration.read_bytes()) + + +def _present(connection: psycopg.Connection[tuple[object, ...]], names: tuple[str, ...]) -> tuple[object, ...]: + return tuple( + row[0] + for row in connection.execute( + "SELECT relname FROM pg_class WHERE relname = ANY(%s) " + "UNION SELECT typname FROM pg_type WHERE typname = ANY(%s) ORDER BY 1", + (list(names), list(names)), + ).fetchall() + ) + + +@pytest.mark.parametrize("old_states", [(), ("applied", "pending", "parked")], ids=["empty", "with_rows"]) +def test_drop_removes_the_old_ledger_and_keeps_spend_log_deliveries(old_states: tuple[str, ...]) -> None: + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + _apply(connection, _OLD_LEDGER) + connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id text PRIMARY KEY)') + _apply(connection, _SPEND_LOG_DELIVERY) + for index, state in enumerate(old_states): + connection.execute( + 'INSERT INTO "LiteLLM_AccountingEvent" (event_id, kind, schema_version, payload_hash, occurred_at, ' + "payload) VALUES (%s, 'cost', 1, 'hash', now(), '{}')", + (f"event-{index}",), + ) + connection.execute( + 'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state) ' + "VALUES (%s, 'LiteLLM_UserTable', 1, %s)", + (f"event-{index}", state), + ) + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, state, accounting_payload, is_batch) ' + "VALUES ('request', 'pending', '{}', false)" + ) + + _apply(connection, _DROP) + _apply(connection, _DROP) + + assert _present(connection, _DROPPED) == () + assert connection.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("request", "pending") + ] diff --git a/tests/integration/database/test_skill_publication.py b/tests/integration/database/test_skill_publication.py new file mode 100644 index 00000000000..476c6bd4399 --- /dev/null +++ b/tests/integration/database/test_skill_publication.py @@ -0,0 +1,201 @@ +import io +import json +import os +import stat +import uuid +import zipfile +from concurrent.futures import ThreadPoolExecutor +from contextlib import AsyncExitStack, ExitStack +from typing import Final + +import psycopg +import pytest +from psycopg import sql + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.plugin_upload_repository import PluginUploadRepository +from tests.integration._support.client import Gateway, list_value, object_value, string_value +from tests.integration._support.database import read_rows + + +def _archive(name: str) -> bytes: + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr( + "SKILL.md", f"---\nname: {name}\ndescription: Published instructions\n---\nRetain these exact bytes.\n", + ) + return buffer.getvalue() + + +def _delete(gateway: Gateway, name: str) -> None: + response: Final = gateway.request("DELETE", f"/claude-code/plugins/{name}") + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_snapshot_failure_rolls_back_catalog_archive_and_history_then_source_replacement_preserves_bytes( + gateway: Gateway, +) -> None: + name: Final = f"integration-version-{uuid.uuid4().hex}" + path: Final = f"/claude-code/plugins/{name}" + history_path: Final = f"/skill-hub/skills/{name}/versions" + content: Final = _archive(name) + async with AsyncExitStack() as cleanup: + created: Final = gateway.request_multipart( + "/claude-code/plugins/upload", + {"metadata": json.dumps({"name": name})}, + {"file": ("skill.zip", content, "application/zip")}, + ) + assert created.status_code == 200, created.text + cleanup.callback(_delete, gateway, name) + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await client.connect() + cleanup.push_async_callback(client.disconnect) + repository: Final = PluginUploadRepository(client) + metadata: Final = await repository.get_metadata(name) + assert metadata is not None + snapshot: Final = (await repository.versions(metadata, 100, 0))[0] + snapshot_bytes: Final = await repository.version_content(metadata, snapshot.id) + assert snapshot_bytes is not None + before: Final = gateway.get(path) + history: Final = gateway.get(history_path) + original: Final = object_value(list_value(history["versions"])[0]) + archive_path: Final = string_value(original["download_url"]) + original_response: Final = gateway.request("GET", archive_path) + assert original_response.status_code == 200, original_response.text + published_content: Final = original_response.content + with zipfile.ZipFile(io.BytesIO(published_content)) as published, zipfile.ZipFile(io.BytesIO(content)) as uploaded: + assert published.read("SKILL.md") == uploaded.read("SKILL.md") + current_path: Final = string_value(object_value(before["source"])["url"]) + original_current: Final = gateway.request("GET", current_path) + assert original_current.status_code == 200, original_current.text + witness: Final = f"publication_{uuid.uuid4().hex}" + trigger: Final = f"{witness}_trigger" + function: Final = f"{witness}_function" + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection, ExitStack() as failure: + connection.execute(sql.SQL("CREATE SEQUENCE {}").format(sql.Identifier(witness))) + failure.callback(connection.execute, sql.SQL("DROP SEQUENCE {}").format(sql.Identifier(witness))) + connection.execute(sql.SQL("GRANT USAGE ON SEQUENCE {} TO PUBLIC").format(sql.Identifier(witness))) + connection.execute( + sql.SQL( + "CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN " + "IF EXISTS (SELECT 1 FROM \"LiteLLM_ClaudeCodePluginTable\" " + "WHERE id = NEW.plugin_id AND name = {}) THEN PERFORM nextval({}); " + "RAISE EXCEPTION 'publication snapshot failure'; END IF; RETURN NEW; END $$" + ).format(sql.Identifier(function), sql.Literal(name), sql.Literal(witness)) + ) + failure.callback(connection.execute, sql.SQL("DROP FUNCTION {}()").format(sql.Identifier(function))) + connection.execute( + sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON "LiteLLM_ClaudeCodePluginVersion" ' + "FOR EACH ROW EXECUTE FUNCTION {}()").format(sql.Identifier(trigger), sql.Identifier(function)) + ) + failure.callback( + connection.execute, + sql.SQL('DROP TRIGGER {} ON "LiteLLM_ClaudeCodePluginVersion"').format(sql.Identifier(trigger)), + ) + rejected: Final = gateway.request( + "PUT", path, {"source": {"source": "github", "repo": "example/changed"}, "description": "must roll back"}, + ) + assert rejected.status_code == 500, rejected.text + assert connection.execute( + sql.SQL("SELECT is_called FROM {}").format(sql.Identifier(witness)) + ).fetchone() == (True,) + assert gateway.get(path) == before + assert gateway.get(history_path) == history + current_archive: Final = gateway.request("GET", current_path) + assert current_archive.status_code == 200 and current_archive.content == original_current.content + updated: Final = gateway.request( + "PUT", path, {"source": {"source": "github", "repo": "example/changed"}}, + ) + assert updated.status_code == 200, updated.text + after: Final = gateway.get(history_path) + versions: Final = tuple(object_value(value) for value in list_value(after["versions"])) + assert [version["version"] for version in versions] == ["1.0.1", "1.0.0"] + assert versions[0]["content_kind"] == "source_reference" + assert versions[1]["content_kind"] == "stored_archive" + retained: Final = gateway.request("GET", archive_path) + assert retained.status_code == 200 and retained.content == published_content + assert read_rows( + 'SELECT plugin_name FROM "LiteLLM_ClaudeCodePluginUpload" WHERE plugin_name=%s', (name,) + ) == [] + assert snapshot in await repository.versions(metadata, 100, 0) + assert await repository.version(metadata, snapshot.id) == snapshot + assert await repository.version_content(metadata, snapshot.id) == snapshot_bytes + hidden: Final = gateway.request("POST", f"{path}/disable") + assert hidden.status_code == 200, hidden.text + assert await repository.versions(metadata, 100, 0) == () + assert await repository.version(metadata, snapshot.id) is None + assert await repository.version_content(metadata, snapshot.id) is None + + +def test_neutral_publication_preserves_portable_files_and_atomic_metadata_history(gateway: Gateway) -> None: + name: Final = f"integration-neutral-{uuid.uuid4().hex}" + path: Final = f"/skill-hub/skills/{name}" + files: Final = { + "SKILL.md": f"---\nname: {name}\ndescription: Portable integration fixture.\n---\n\nRead references/notes.md.\n", + "references/notes.md": "Preserve the original supporting file.\n", + "scripts/check.sh": "#!/bin/sh\nprintf 'portable\\n'\n", + } + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for filename, content in files.items(): + entry: Final = zipfile.ZipInfo(filename) + entry.create_system = 3 + entry.external_attr = (stat.S_IFREG | (0o755 if filename.endswith(".sh") else 0o644)) << 16 + archive.writestr(entry, content) + with ExitStack() as cleanup: + created: Final = gateway.request_multipart( + "/skill-hub/skills/upload", {}, {"file": ("skill.zip", buffer.getvalue(), "application/zip")}, + ) + assert created.status_code == 200, created.text + cleanup.callback(_delete, gateway, name) + assert created.json()["version"] == "1.0.0" + before: Final = gateway.get(path) + assert string_value(before["updated_at"]) == created.json()["updated_at"] + history_before: Final = gateway.get(f"{path}/versions") + first: Final = object_value(list_value(history_before["versions"])[0]) + version_id: Final = string_value(first["id"]) + archive_path: Final = f"{path}/versions/{version_id}/archive" + downloaded: Final = gateway.request("GET", archive_path) + assert downloaded.status_code == 200, downloaded.text + with zipfile.ZipFile(io.BytesIO(downloaded.content)) as portable: + assert {member: portable.read(member).decode() for member in portable.namelist()} == files + assert stat.S_IMODE(portable.getinfo("scripts/check.sh").external_attr >> 16) == 0o755 + assert gateway.get(f"{path}/versions/{version_id}") == first + + def publish(index: int) -> int: + response: Final = gateway.request( + "PUT", path, + {"description": f"Registry edit {index}", "expected_updated_at": before["updated_at"]}, + ) + if response.status_code == 200: + assert string_value(response.json()["updated_at"]) != before["updated_at"] + return response.status_code + + with ThreadPoolExecutor(max_workers=2) as pool: + statuses: Final = tuple(pool.map(publish, range(2))) + assert sorted(statuses) == [200, 409] + assert publish(2) == 409 + current: Final = gateway.get(path) + history_after: Final = gateway.get(f"{path}/versions") + versions: Final = tuple(object_value(value) for value in list_value(history_after["versions"])) + assert [version["version"] for version in versions] == ["1.0.1", "1.0.0"] + latest: Final = versions[0] + manifest: Final = object_value(latest["manifest"]) + assert current["version"] == manifest["version"] == latest["version"] + assert current["description"] == manifest["description"] == latest["description"] + assert manifest["agent_skill"] == object_value(first["manifest"])["agent_skill"] + stored: Final = read_rows( + 'SELECT version, description, manifest_json FROM "LiteLLM_ClaudeCodePluginTable" WHERE name=%s', (name,) + ) + assert len(stored) == 1 + assert stored[0]["version"] == current["version"] + assert json.loads(string_value(stored[0]["manifest_json"])) == manifest + retained: Final = gateway.request("GET", archive_path) + latest_archive: Final = gateway.request("GET", f"{path}/versions/{latest['id']}/archive") + assert retained.content == latest_archive.content == downloaded.content + rejected: Final = gateway.request("PUT", path, {"version": "1.0.0", "description": "rejected"}) + assert rejected.status_code == 400, rejected.text + assert gateway.get(path) == current + assert gateway.get(f"{path}/versions") == history_after diff --git a/tests/integration/database/test_skill_submissions.py b/tests/integration/database/test_skill_submissions.py new file mode 100644 index 00000000000..cc00bf6e359 --- /dev/null +++ b/tests/integration/database/test_skill_submissions.py @@ -0,0 +1,454 @@ +import asyncio +import hashlib +import io +import os +import uuid +import zipfile +from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass, replace +from typing import Final, Literal + +import psycopg +import pytest +import pytest_asyncio +from pydantic import JsonValue, TypeAdapter + +from litellm.models.plugin_upload import ( + PluginPublication, + PluginPublicationBase, + PluginPublicationError, + PluginUploadStore, + PublishedPlugin, +) +from litellm.models.skill_submission import ( + SkillSubmission, + SkillSubmissionStore, + SubmissionConflict, + SubmissionLimitExceeded, + SubmissionReviewed, +) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.plugin_upload_repository import PluginUploadRepository +from litellm.repositories.skill_submission_repository import SkillSubmissionRepository +from tests.integration._support.client import eventually +from tests.integration._support.database import read_rows, write_rows + + +@dataclass(frozen=True, slots=True) +class PendingUpdate: + publications: PluginUploadStore + submissions: SkillSubmissionStore + original: PluginPublication + proposal: PluginPublication + base: PublishedPlugin + submission: SkillSubmission + + +def _publication(name: str, description: str, base_url: str = "https://skills.example.test") -> PluginPublication: + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("SKILL.md", f"---\nname: {name}\ndescription: {description}\n---\n{description}\n") + content: Final = buffer.getvalue() + digest: Final = hashlib.sha256(content).hexdigest() + return PluginPublication( + name=name, + version=None, + description=description, + manifest={ + "name": name, + "description": description, + "extension": {"retained": True}, + "source": { + "source": "archive", + "url": f"{base_url}/claude-code/plugins/{name}/archive/{digest}.zip", + "sha256": digest, + }, + }, + content=content, + created_by=f"{name}-owner", + ) + + +def _published_rows(name: str) -> Sequence[Mapping[str, JsonValue]]: + return read_rows( + """SELECT 'catalog' AS storage, to_jsonb(p) AS value + FROM "LiteLLM_ClaudeCodePluginTable" p WHERE name=%s + UNION ALL + SELECT 'archive', to_jsonb(a) FROM "LiteLLM_ClaudeCodePluginUpload" a WHERE plugin_name=%s + UNION ALL + SELECT 'history', to_jsonb(v) FROM "LiteLLM_ClaudeCodePluginVersion" v + JOIN "LiteLLM_ClaudeCodePluginTable" p ON p.id=v.plugin_id WHERE p.name=%s + ORDER BY storage, value""", + (name, name, name), + ) + + +@pytest_asyncio.fixture(loop_scope="function") +async def pending_update() -> AsyncIterator[PendingUpdate]: + name: Final = f"integration-submission-{uuid.uuid4().hex}" + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + publications: Final = PluginUploadRepository(client) + submissions: Final = SkillSubmissionRepository(client) + original: Final = _publication(name, "Published instructions") + base: Final = await publications.publish(original) + assert isinstance(base, PublishedPlugin) + proposal: Final = _publication(name, "Proposed instructions", "https://member.example.test") + submission: Final = await submissions.submit( + proposal, f"{name}-owner", PluginPublicationBase(base.id, base.updated_at) + ) + assert isinstance(submission, SkillSubmission) + yield PendingUpdate(publications, submissions, original, proposal, base, submission) + finally: + try: + write_rows('DELETE FROM "LiteLLM_SkillSubmission" WHERE plugin_name=%s', (name,)) + write_rows('DELETE FROM "LiteLLM_ClaudeCodePluginTable" WHERE name=%s', (name,)) + finally: + await client.disconnect() + + +async def test_approval_commits_version_archive_history_and_review(pending_update: PendingUpdate) -> None: + case: Final = pending_update + before: Final = await case.publications.get_metadata(case.original.name) + assert before is not None + assert await case.publications.get_content(before) == case.original.content + assert case.submission.proposal.version is None + reviewed: Final = await case.submissions.review( + case.submission.id, "integration-reviewer", True, None, publication_base_url="https://skills.example.test" + ) + assert isinstance(reviewed, SubmissionReviewed) + assert reviewed.plugin is not None and reviewed.plugin.version == "1.0.1" + stored: Final = await case.submissions.get(case.submission.id, None) + assert stored is not None + assert (stored.status, stored.reviewed_by, stored.rejection_reason) == ( + "approved", + "integration-reviewer", + None, + ) + assert stored.reviewed_at is not None + assert stored.proposal == case.submission.proposal + assert await case.submissions.get_archive(stored.id, None) == case.proposal.content + current: Final = await case.publications.get_plugin(case.original.name) + assert current is not None + published_manifest: Final = { + **case.proposal.manifest, + "source": { + **TypeAdapter(dict[str, str]).validate_python(case.submission.proposal.manifest["source"]), + "url": f"https://skills.example.test/claude-code/plugins/{case.original.name}/archive/" + f"{hashlib.sha256(case.proposal.content or b'').hexdigest()}.zip", + }, + } + assert (current.id, current.version, current.description, current.manifest) == ( + case.base.id, + reviewed.plugin.version, + case.proposal.description, + {**published_manifest, "version": reviewed.plugin.version}, + ) + metadata: Final = await case.publications.get_metadata(case.original.name) + assert metadata is not None + assert await case.publications.get_content(metadata) == case.proposal.content + history: Final = await case.publications.versions(metadata, 10, 0) + assert tuple((version.version, version.description, version.archive_available) for version in history) == ( + ("1.0.1", case.proposal.description, True), + ("1.0.0", case.original.description, True), + ) + for version, publication, manifest in zip( + history, (case.proposal, case.original), (published_manifest, case.original.manifest), strict=True + ): + assert version.manifest == {**manifest, "version": version.version} + assert version.published_by == publication.created_by + assert await case.publications.version_content(metadata, version.id) == publication.content + + +async def test_simultaneous_approve_reject_has_one_winner(pending_update: PendingUpdate) -> None: + case: Final = pending_update + before: Final = _published_rows(case.original.name) + with psycopg.connect(os.environ["DATABASE_URL"]) as blocker: + blocker.execute('SELECT id FROM "LiteLLM_SkillSubmission" WHERE id=%s FOR UPDATE', (case.submission.id,)) + blocker_pid: Final = str(blocker.info.backend_pid) + reviews: Final = ( + asyncio.create_task( + case.submissions.review( + case.submission.id, + "integration-approver", + True, + None, + publication_base_url="https://skills.example.test", + ) + ), + asyncio.create_task( + case.submissions.review(case.submission.id, "integration-rejector", False, "Needs revision") + ), + ) + try: + await asyncio.to_thread( + eventually, + lambda: read_rows( + """WITH RECURSIVE blocked AS ( + SELECT %s::integer AS pid + UNION + SELECT a.pid FROM pg_stat_activity a + JOIN blocked b ON b.pid = ANY(pg_blocking_pids(a.pid)) + ) SELECT pid FROM blocked WHERE pid<>%s::integer""", + (blocker_pid, blocker_pid), + ), + lambda rows: len(rows) == 2, + seconds=3, + ) + finally: + blocker.rollback() + results: Final = await asyncio.gather(*reviews) + winners: Final = tuple(result for result in results if isinstance(result, SubmissionReviewed)) + assert len(winners) == 1 + assert sum(isinstance(result, SubmissionConflict) for result in results) == 1 + stored: Final = await case.submissions.get(case.submission.id, None) + assert stored is not None and stored.reviewed_at is not None + winner: Final = winners[0] + assert (stored.status, stored.reviewed_by, stored.rejection_reason) == ( + winner.submission.status, + winner.submission.reviewed_by, + winner.submission.rejection_reason, + ) + if isinstance(results[0], SubmissionConflict): + assert (stored.status, stored.reviewed_by, stored.rejection_reason) == ( + "rejected", + "integration-rejector", + "Needs revision", + ) + assert _published_rows(case.original.name) == before + return + assert (stored.status, stored.reviewed_by, stored.rejection_reason) == ( + "approved", + "integration-approver", + None, + ) + metadata: Final = await case.publications.get_metadata(case.original.name) + assert metadata is not None + assert await case.publications.get_content(metadata) == case.proposal.content + assert tuple(version.version for version in await case.publications.versions(metadata, 10, 0)) == ( + "1.0.1", + "1.0.0", + ) + + +async def test_publication_failure_rolls_back_and_leaves_submission_pending(pending_update: PendingUpdate) -> None: + case: Final = pending_update + collision_id: Final = str(uuid.uuid4()) + write_rows( + """INSERT INTO "LiteLLM_ClaudeCodePluginVersion" (id, plugin_id, version) + VALUES (%s, %s, '1.0.1')""", + (collision_id, case.base.id), + ) + before: Final = _published_rows(case.original.name) + failed: Final = await case.submissions.review( + case.submission.id, "integration-reviewer", True, None, publication_base_url="https://skills.example.test" + ) + assert isinstance(failed, SubmissionConflict) + assert _published_rows(case.original.name) == before + assert await case.submissions.get(case.submission.id, None) == case.submission + assert await case.submissions.get_archive(case.submission.id, None) == case.proposal.content + write_rows('DELETE FROM "LiteLLM_ClaudeCodePluginVersion" WHERE id=%s', (collision_id,)) + retried: Final = await case.submissions.review( + case.submission.id, "integration-reviewer", True, None, publication_base_url="https://skills.example.test" + ) + assert isinstance(retried, SubmissionReviewed) + assert retried.plugin is not None and retried.plugin.version == "1.0.1" + stored: Final = await case.submissions.get(case.submission.id, None) + assert stored is not None and stored.status == "approved" + + +@pytest.mark.parametrize("change", ("update", "recreate")) +async def test_stale_target_cannot_publish( + pending_update: PendingUpdate, change: Literal["update", "recreate"] +) -> None: + case: Final = pending_update + intervening: Final = _publication(case.original.name, "Intervening publication") + if change == "recreate": + write_rows('DELETE FROM "LiteLLM_ClaudeCodePluginTable" WHERE name=%s', (case.original.name,)) + replacement: Final = await case.publications.publish( + intervening, + PluginPublicationBase(case.base.id, case.base.updated_at) if change == "update" else None, + ) + assert isinstance(replacement, PublishedPlugin) + before: Final = _published_rows(case.original.name) + reviewed: Final = await case.submissions.review( + case.submission.id, "integration-reviewer", True, None, publication_base_url="https://skills.example.test" + ) + assert isinstance(reviewed, PluginPublicationError) and reviewed.kind == "conflict" + assert _published_rows(case.original.name) == before + assert await case.submissions.get(case.submission.id, None) == case.submission + + +async def test_concurrent_submissions_share_user_limit_and_review_releases_capacity( + pending_update: PendingUpdate, +) -> None: + case: Final = pending_update + owner: Final = case.submission.submitted_by + base: Final = PluginPublicationBase(case.base.id, case.base.updated_at) + for _ in range(18): + assert isinstance(await case.submissions.submit(case.proposal, owner, base), SkillSubmission) + peer: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await peer.connect() + try: + with psycopg.connect(os.environ["DATABASE_URL"]) as blocker: + blocker.execute( + "SELECT pg_advisory_xact_lock(hashtext('litellm.skill_submissions'), hashtext(%s))", (owner,) + ) + blocker_pid: Final = str(blocker.info.backend_pid) + requests: Final = ( + asyncio.create_task(case.submissions.submit(case.proposal, owner, base)), + asyncio.create_task(SkillSubmissionRepository(peer).submit(case.proposal, owner, base)), + ) + try: + await asyncio.to_thread( + eventually, + lambda: read_rows( + "SELECT pid FROM pg_stat_activity WHERE %s::integer = ANY(pg_blocking_pids(pid))", + (blocker_pid,), + ), + lambda rows: len(rows) == 2, + seconds=3, + ) + finally: + blocker.rollback() + results: Final = await asyncio.gather(*requests) + finally: + await peer.disconnect() + assert sum(isinstance(result, SkillSubmission) for result in results) == 1 + assert sum(isinstance(result, SubmissionLimitExceeded) for result in results) == 1 + full: Final = await case.submissions.list(owner, "pending_review", 20, 0) + assert (len(full.submissions), full.has_more) == (20, False) + other: Final = await case.submissions.submit(case.proposal, f"{owner}-other", base) + assert isinstance(other, SkillSubmission) + rejected: Final = await case.submissions.review( + case.submission.id, "integration-reviewer", False, "Review complete" + ) + assert isinstance(rejected, SubmissionReviewed) + assert await case.submissions.get_archive(case.submission.id, owner) == case.proposal.content + replacement: Final = await case.submissions.submit(case.proposal, owner, base) + assert isinstance(replacement, SkillSubmission) + after: Final = await case.submissions.list(owner, "pending_review", 20, 0) + assert len(after.submissions) == 20 + assert all(row.id != case.submission.id and row.submitted_by == owner for row in after.submissions) + + +async def test_submission_pages_filter_owner_and_status_before_limit(pending_update: PendingUpdate) -> None: + case: Final = pending_update + owner: Final = case.submission.submitted_by + other: Final = f"{owner}-other" + generated: Final = await asyncio.gather( + *( + case.submissions.submit(case.proposal, identity, PluginPublicationBase(case.base.id, case.base.updated_at)) + for identity in (owner, other, owner, other) + ) + ) + submitted: Final = tuple(result for result in generated if isinstance(result, SkillSubmission)) + assert len(submitted) == 4 + rejected: Final = await case.submissions.review(submitted[0].id, "integration-reviewer", False, None) + assert isinstance(rejected, SubmissionReviewed) + write_rows( + """UPDATE "LiteLLM_SkillSubmission" SET created_at=TIMESTAMP '9999-01-01' + WHERE plugin_name=%s""", + (case.original.name,), + ) + own_pending: Final = tuple(sorted((case.submission.id, submitted[2].id), reverse=True)) + first: Final = await case.submissions.list(owner, "pending_review", 1, 0) + second: Final = await case.submissions.list(owner, "pending_review", 1, 1) + empty: Final = await case.submissions.list(owner, "pending_review", 1, 2) + assert (tuple(row.id for row in first.submissions), first.has_more) == (own_pending[:1], True) + assert (tuple(row.id for row in second.submissions), second.has_more) == (own_pending[1:], False) + assert (empty.submissions, empty.has_more) == ((), False) + reviewed: Final = await case.submissions.list(owner, "rejected", 20, 0) + assert (tuple(row.id for row in reviewed.submissions), reviewed.has_more) == ((submitted[0].id,), False) + own_all: Final = await case.submissions.list(owner, None, 20, 0) + assert tuple(row.id for row in own_all.submissions) == tuple(sorted((*own_pending, submitted[0].id), reverse=True)) + other_page: Final = await case.submissions.list(other, None, 20, 0) + assert tuple(row.id for row in other_page.submissions) == tuple( + sorted((submitted[1].id, submitted[3].id), reverse=True) + ) + global_pending: Final = await case.submissions.list(None, "pending_review", 3, 0) + all_pending: Final = tuple(sorted((*own_pending, submitted[1].id, submitted[3].id), reverse=True)) + assert (tuple(row.id for row in global_pending.submissions), global_pending.has_more) == (all_pending[:3], True) + + +@pytest.mark.parametrize("completion_seconds", (13, 21), ids=("before-deadline", "at-deadline")) +async def test_scan_claim_retry_and_late_result_preserve_proposal( + pending_update: PendingUpdate, completion_seconds: int, +) -> None: + from datetime import datetime, timedelta, timezone + + from litellm.models.skill_submission import SkillScanInput + from litellm.types.proxy.skill_scanning import SkillScan, SkillScanReport + + case: Final = pending_update + now: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + first: Final = SkillScan( + id="first", status="running", requested_by="reviewer", started_at=now, + expires_at=now + timedelta(seconds=10), + ) + started: Final = await case.submissions.start_scan(case.submission.id, first) + assert isinstance(started, SkillScanInput) + assert started.content == case.proposal.content + assert started.scan.archive_sha256 == hashlib.sha256(case.proposal.content or b"").hexdigest() + duplicate: Final = await case.submissions.start_scan(case.submission.id, first.model_copy(update={"id": "duplicate"})) + assert isinstance(duplicate, SubmissionConflict) + second: Final = first.model_copy(update={ + "id": "second", "started_at": now + timedelta(seconds=11), "expires_at": now + timedelta(seconds=21), + }) + retry: Final = await case.submissions.start_scan(case.submission.id, second) + assert isinstance(retry, SkillScanInput) + report: Final = SkillScanReport( + scanner_version="test-version", complete=False, scan_mode="static-only", risk_score=90, findings=(), + ) + await case.submissions.finish_scan(case.submission.id, started.scan.model_copy(update={ + "status": "completed", "report": report, "completed_at": now + timedelta(seconds=12), + })) + assert await case.submissions.get_scan(case.submission.id) == retry.scan + completed: Final = retry.scan.model_copy(update={ + "status": "completed", "report": report, "completed_at": now + timedelta(seconds=completion_seconds), + }) + expected: Final = completed if completion_seconds < 21 else retry.scan.at(now + timedelta(seconds=21)) + await case.submissions.finish_scan(case.submission.id, completed) + assert await case.submissions.get_scan(case.submission.id) == expected + assert await case.submissions.get(case.submission.id, None) == case.submission + approved: Final = await case.submissions.review(case.submission.id, "reviewer", True, None, publication_base_url="https://gateway.test") + assert isinstance(approved, SubmissionReviewed) + assert await case.submissions.get_scan(case.submission.id) == expected + denied: Final = await case.submissions.start_scan(case.submission.id, second) + assert isinstance(denied, SubmissionConflict) + + +@pytest.mark.parametrize("available, remote", ((True, False), (False, False), (False, True))) +async def test_metadata_scan_requires_the_matching_managed_archive( + pending_update: PendingUpdate, available: bool, remote: bool +) -> None: + from datetime import datetime, timedelta, timezone + + from litellm.models.skill_submission import SkillScanInput + from litellm.types.proxy.skill_scanning import SkillScan + + case: Final = pending_update + submission: Final = await case.submissions.submit( + replace(case.original, content=None, manifest={ + **case.original.manifest, + **({"source": {"source": "archive", "url": "https://example.org/skill.zip"}} if remote else {}), + }), "member", PluginPublicationBase(case.base.id, case.base.updated_at) + ) + if not available: + write_rows('DELETE FROM "LiteLLM_ClaudeCodePluginUpload" WHERE plugin_name=%s', (case.original.name,)) + now: Final = datetime.now(timezone.utc) + result: Final = await case.submissions.start_scan(submission.id, SkillScan( + id="metadata-scan", status="running", requested_by="admin", started_at=now, + expires_at=now + timedelta(seconds=10), + )) + if available or remote: + assert isinstance(result, SkillScanInput) + assert result.content == (None if remote else case.original.content) + if remote: + assert result.source == {"source": "archive", "url": "https://example.org/skill.zip"} + assert result.scan.archive_sha256 is None + else: + assert isinstance(result, SubmissionConflict) + assert await case.submissions.get_scan(submission.id) is None diff --git a/tests/integration/database/test_spendlogs_delivery.py b/tests/integration/database/test_spendlogs_delivery.py new file mode 100644 index 00000000000..19f01a6b902 --- /dev/null +++ b/tests/integration/database/test_spendlogs_delivery.py @@ -0,0 +1,512 @@ +import asyncio +import os +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import psycopg +import pytest +import pytest_asyncio +from prisma import Prisma + +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerTransaction, +) +from litellm.repositories.spend_log_insert import spend_log_insert_statement +from tests.integration._support.database import scratch_database + +_MIGRATIONS: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/migrations" + + +@pytest.fixture +def delivery_database(request: pytest.FixtureRequest) -> Iterator[psycopg.Connection[tuple[object, ...]]]: + with scratch_database("spendlogs_delivery") as url, psycopg.connect(url, autocommit=True) as connection: + if getattr(request, "param", "public") == "ledger_tenant": + connection.execute("CREATE SCHEMA ledger_tenant") + connection.execute("SET search_path TO ledger_tenant") + connection.execute((_MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql").read_text()) + connection.execute("""CREATE TABLE "LiteLLM_SpendLogs" ( + request_id text PRIMARY KEY, call_type text NOT NULL, spend float8 NOT NULL DEFAULT 0, + status text, + metadata jsonb DEFAULT '{}', "startTime" timestamp(3) NOT NULL, + created_at timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP + )""") + connection.execute((_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text()) + yield connection + + +def _insert( + connection: psycopg.Connection[tuple[object, ...]], rows: Sequence[Mapping[str, object]], *, ledger: bool = True +) -> None: + from psycopg.types.json import Jsonb + + statement: Final = spend_log_insert_statement(rows) + with connection.transaction(): + inserted: Final = { + row[0] for row in connection.execute(statement[0].replace("$1", "%s"), (statement[1],)).fetchall() + } + for row in rows: + if ledger and row["request_id"] in inserted: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) ' + "VALUES (%s, %s, %s, %s)", + (row["request_id"], row["request_id"], Jsonb(dict(row)), row.get("call_type") == "aretrieve_batch"), + ) + + +def _with_payload(row: Mapping[str, object]) -> Mapping[str, object]: + return {**row, "accounting_payload": dict(row)} + + +def _row(request_id: str) -> Mapping[str, object]: + return { + "request_id": request_id, + "call_type": "acompletion", + "spend": 0.5, + "startTime": "2026-11-02T12:00:00+00:00", + "metadata": '{"accounting_savings":{"compression":0.2}}', + } + + +@pytest.mark.parametrize("call_type", ["acompletion", "aretrieve_batch"]) +@pytest.mark.parametrize("mode", ["local", "url", "disabled"]) +@pytest.mark.parametrize("partitioned", [False, True]) +@pytest.mark.asyncio +async def test_delivery_first_acceptance_deduplicates_all_storage_modes( + delivery_database, delivery_client, mode, partitioned, call_type +): + from types import SimpleNamespace + + from litellm.repositories.table_repositories import SpendLogsRepository + + if partitioned: + delivery_database.execute('ALTER TABLE "LiteLLM_SpendLogs" RENAME TO spend_template') + delivery_database.execute( + 'CREATE TABLE "LiteLLM_SpendLogs" (LIKE spend_template INCLUDING DEFAULTS INCLUDING CONSTRAINTS, PRIMARY KEY(request_id, "startTime")) PARTITION BY RANGE ("startTime")' + ) + delivery_database.execute('CREATE TABLE "LiteLLM_SpendLogs_p20261102" PARTITION OF "LiteLLM_SpendLogs" DEFAULT') + delivery_database.execute("DROP TABLE spend_template") + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + row = {**_with_payload({**_row("shared"), "call_type": call_type}), "accounting_log_disabled": mode == "disabled"} + results = await asyncio.gather( + repository.insert_with_deliveries((row,), write_spend_logs=mode != "url"), + repository.insert_with_deliveries( + ({**row, "startTime": "2026-11-03T12:00:00"},), write_spend_logs=mode != "url" + ), + ) + assert sorted(results) == [0, 1] + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(mode == "local"),) + assert delivery_database.execute( + "SELECT accounting_payload->>'spend', state::text FROM \"LiteLLM_SpendLogDelivery\"" + ).fetchall() == [("0.5", "pending")] + if partitioned: + delivery_database.execute('DROP TABLE "LiteLLM_SpendLogs_p20261102"') + else: + delivery_database.execute('DELETE FROM "LiteLLM_SpendLogs"') + + async def apply(transaction, deliveries): + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(deliveries[0].row["spend"])) + + worker = AccountingDeliveryRepository(delivery_client) + assert (await worker.apply_pending(apply)).applied == 1 + assert (await worker.apply_pending(apply)).applied == 0 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.parametrize("mode", ["local", "url", "disabled"]) +@pytest.mark.asyncio +async def test_duplicate_rows_in_one_flush_keep_first_accounting_payload(delivery_database, delivery_client, mode): + from types import SimpleNamespace + + from litellm.repositories.table_repositories import SpendLogsRepository + + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + first = {**_with_payload(_row("repeated")), "accounting_log_disabled": mode == "disabled"} + later = { + **_with_payload({**_row("repeated"), "spend": 9.0, "startTime": "2026-11-03T12:00:00+00:00"}), + "accounting_log_disabled": mode == "disabled", + } + assert await repository.insert_with_deliveries((first, later), write_spend_logs=mode != "url") == 1 + assert delivery_database.execute( + 'SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"' + ).fetchone() == (first["accounting_payload"],) + assert delivery_database.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == ( + [(first["spend"],)] if mode == "local" else [] + ) + + +@pytest.mark.parametrize(("legacy_spend", "accepted"), [(0.5, 0), (0.0, 1)]) +@pytest.mark.asyncio +async def test_batch_charged_before_the_ledger_is_not_charged_again( + delivery_database, delivery_client, legacy_spend, accepted +): + from types import SimpleNamespace + + from litellm.repositories.table_repositories import SpendLogsRepository + + batch: Final = {**_row("batch_legacy"), "call_type": "aretrieve_batch", "status": "success"} + _insert(delivery_database, ({**batch, "spend": legacy_spend},), ledger=False) + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + assert await repository.insert_with_deliveries((_with_payload({**batch, "spend": 1.0}),)) == accepted + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (accepted,) + assert delivery_database.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [(legacy_spend,)] + + +@pytest.mark.parametrize("table", ["LiteLLM_SpendLogs", "LiteLLM_SpendLogDelivery"]) +@pytest.mark.asyncio +async def test_transaction_a_rolls_back_payload_and_log_on_failure(delivery_database, delivery_client, table): + from types import SimpleNamespace + + from prisma.errors import PrismaError + from psycopg import sql + + from litellm.repositories.table_repositories import SpendLogsRepository + + delivery_database.execute( + sql.SQL("ALTER TABLE {} ADD CONSTRAINT reject_poison CHECK (request_id <> 'poison')").format( + sql.Identifier(table) + ) + ) + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + with pytest.raises(PrismaError): + await repository.insert_with_deliveries(tuple(_with_payload(_row(name)) for name in ("good", "poison"))) + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,) + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,) + + +@pytest.mark.asyncio +async def test_worker_rolls_back_destinations_and_parks_only_isolated_failure( + delivery_database: psycopg.Connection[tuple[object, ...]], +) -> None: + url: Final = urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}")) + client: Final = Prisma(datasource={"url": url}) + delivery_database.execute("CREATE TABLE totals (spend float8 NOT NULL)") + delivery_database.execute("INSERT INTO totals VALUES (0)") + _insert(delivery_database, (_row("bad"), _row("good"))) + + async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw( + "UPDATE totals SET spend = spend + $1::float8", sum(float(str(row.row["spend"])) for row in rows) + ) + if any(row.row["request_id"] == "bad" for row in rows): + raise ValueError("Invalid projected destination") + + await client.connect() + try: + repository: Final = AccountingDeliveryRepository(client) + failed: Final = await repository.apply_pending(apply, max_attempts=2) + assert failed.failed == 2 + assert failed.parked == () + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,) + assert delivery_database.execute( + 'SELECT DISTINCT state::text, attempts FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("pending", 1)] + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP') + isolated: Final = await repository.apply_pending(apply, max_attempts=2) + assert isolated.parked == ("bad",) + applied: Final = await repository.apply_pending(apply, max_attempts=2) + assert applied.applied == 1 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + assert delivery_database.execute( + 'SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id' + ).fetchall() == [("bad", "parked"), ("good", "applied")] + assert (await repository.apply_pending(apply)).applied == 0 + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_a_locked_database_never_parks_a_valid_delivery(delivery_database, delivery_client): + _insert(delivery_database, (_row("valid"),)) + + async def apply(transaction, deliveries): + await transaction.execute_raw("SET LOCAL lock_timeout = '50ms'") + await transaction.execute_raw( + "UPDATE totals SET spend = spend + $1::float8", sum(float(row.row["spend"]) for row in deliveries) + ) + + repository = AccountingDeliveryRepository(delivery_client) + with psycopg.connect( + urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}")) + ) as locker: + locker.execute("LOCK TABLE totals IN ACCESS EXCLUSIVE MODE") + for _ in range(4): + result = await repository.apply_pending(apply, max_attempts=2) + assert result.failed == 1 and result.parked == () + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP') + assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("pending", 4) + ] + assert (await repository.apply_pending(apply, max_attempts=2)).applied == 1 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_an_unreachable_database_reported_as_a_data_error_never_parks_a_delivery( + delivery_database, delivery_client +): + from prisma.errors import DataError + + _insert(delivery_database, (_row("valid"),)) + + async def unreachable(transaction, deliveries): + raise DataError(data={"user_facing_error": {"message": "Can't reach database server at `db`:`5432`"}}) + + repository = AccountingDeliveryRepository(delivery_client) + for _ in range(3): + assert (await repository.apply_pending(unreachable, max_attempts=2)).parked == () + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP') + assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("pending", 3) + ] + + +@pytest_asyncio.fixture(loop_scope="function") +async def delivery_client(delivery_database: psycopg.Connection[tuple[object, ...]]) -> AsyncIterator[Prisma]: + url: Final = urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}")) + client: Final = Prisma(datasource={"url": url}) + delivery_database.execute("CREATE TABLE totals (spend float8 NOT NULL)") + delivery_database.execute("INSERT INTO totals VALUES (0)") + await client.connect() + try: + yield client + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_two_workers_claim_disjoint_rows_and_apply_once( + delivery_database: psycopg.Connection[tuple[object, ...]], + delivery_client: Prisma, +) -> None: + _insert(delivery_database, (_row("first"), _row("second"))) + entered: Final[asyncio.Queue[str]] = asyncio.Queue() + release: Final = asyncio.Event() + + async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + assert len(rows) == 1 + await entered.put(str(rows[0].row["request_id"])) + await release.wait() + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + + repository: Final = AccountingDeliveryRepository(delivery_client) + first: Final = asyncio.create_task(repository.apply_pending(apply, batch_size=1)) + first_id: Final = await asyncio.wait_for(entered.get(), timeout=10) + second: Final = asyncio.create_task(repository.apply_pending(apply, batch_size=1)) + try: + second_id: Final = await asyncio.wait_for(entered.get(), timeout=10) + assert first_id != second_id + finally: + release.set() + results: Final = await asyncio.gather(first, second) + assert sum(result.applied for result in results) == 2 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (1.0,) + assert delivery_database.execute( + "SELECT COUNT(*) FROM \"LiteLLM_SpendLogDelivery\" WHERE state = 'applied'" + ).fetchone() == (2,) + + +@pytest.mark.asyncio +async def test_cancelled_worker_rolls_back_and_restart_applies_once( + delivery_database: psycopg.Connection[tuple[object, ...]], + delivery_client: Prisma, +) -> None: + _insert(delivery_database, (_row("restart"),)) + wrote: Final = asyncio.Event() + hold: Final = asyncio.Event() + + async def interrupted(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + wrote.set() + await hold.wait() + + repository: Final = AccountingDeliveryRepository(delivery_client) + task: Final = asyncio.create_task(repository.apply_pending(interrupted)) + await asyncio.wait_for(wrote.wait(), timeout=10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,) + assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchone() == ( + "pending", + 0, + ) + + async def complete(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + + assert (await repository.apply_pending(complete)).applied == 1 + assert (await repository.apply_pending(complete)).applied == 0 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_worker_applies_accounting_after_spend_logs_are_deleted( + delivery_database: psycopg.Connection[tuple[object, ...]], delivery_client: Prisma +) -> None: + _insert(delivery_database, (_row("removed"),)) + delivery_database.execute('DELETE FROM "LiteLLM_SpendLogs"') + + async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + + repository: Final = AccountingDeliveryRepository(delivery_client) + assert (await repository.apply_pending(apply)).applied == 1 + assert (await repository.apply_pending(apply)).applied == 0 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_receipt_cleanup_keeps_pending_and_parked_rows_and_reports_lag( + delivery_database: psycopg.Connection[tuple[object, ...]], delivery_client: Prisma +) -> None: + _insert(delivery_database, tuple(_row(state) for state in ("pending", "parked", "applied"))) + delivery_database.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" ' + 'SET state = request_id::"AccountingDeliveryState", ' + "created_at = CURRENT_TIMESTAMP - INTERVAL '40 days', " + "applied_at = CURRENT_TIMESTAMP - INTERVAL '40 days'" + ) + repository = AccountingDeliveryRepository(delivery_client) + assert await repository.cleanup_applied() == 1 + metrics = await repository.metrics() + assert metrics.pending_count == 1 + assert metrics.parked_count == 1 + assert metrics.oldest_pending_seconds >= 40 * 86400 + assert delivery_database.execute( + 'SELECT request_id FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id' + ).fetchall() == [("parked",), ("pending",)] + + +@pytest.mark.parametrize("delivery_database", ["public", "ledger_tenant"], indirect=True) +@pytest.mark.parametrize("state", [None, "pending", "parked", "applied"]) +def test_delivery_migration_replay_preserves_ledger_receipts(delivery_database, state): + if state is not None: + _insert(delivery_database, (_row("replay"),)) + delivery_database.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" SET state=%s, attempts=3, last_error=%s', + (state, "retained failure"), + ) + before = delivery_database.execute('SELECT * FROM "LiteLLM_SpendLogDelivery"').fetchall() + migration = (_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text() + delivery_database.execute(migration) + delivery_database.execute(migration) + assert delivery_database.execute('SELECT * FROM "LiteLLM_SpendLogDelivery"').fetchall() == before + assert delivery_database.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(state is not None),) + + +@pytest.mark.parametrize("state", ["pending", "parked"]) +def test_delivery_migration_preserves_unused_old_receipts(state): + with scratch_database("spendlogs_old_delivery") as url, psycopg.connect(url, autocommit=True) as connection: + connection.execute((_MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql").read_text()) + connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id text PRIMARY KEY)') + connection.execute( + 'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state) VALUES (%s, %s, 1, %s)', + ("old-event", "LiteLLM_UserTable", state), + ) + connection.execute((_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text()) + assert connection.execute('SELECT event_id, state::text FROM "LiteLLM_AccountingDelivery"').fetchall() == [ + ("old-event", state) + ] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,) + + +@pytest.mark.asyncio +async def test_malformed_payload_is_parked_without_losing_other_deliveries(delivery_database, delivery_client): + _insert(delivery_database, (_row("bad"), _row("good"))) + delivery_database.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" SET accounting_payload=$$ [] $$::jsonb WHERE request_id=$$bad$$' + ) + + async def apply(transaction, deliveries): + await transaction.execute_raw( + "UPDATE totals SET spend = spend + $1::float8", sum(float(row.row["spend"]) for row in deliveries) + ) + + repository = AccountingDeliveryRepository(delivery_client) + assert (await repository.apply_pending(apply, max_attempts=2)).failed == 2 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,) + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at=CURRENT_TIMESTAMP') + assert (await repository.apply_pending(apply, max_attempts=2)).parked == ("bad",) + assert (await repository.apply_pending(apply, max_attempts=2)).applied == 1 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_retention_keeps_batch_receipts_indefinitely(delivery_database, delivery_client): + _insert(delivery_database, (_row("normal"), {**_row("batch"), "call_type": "aretrieve_batch"})) + delivery_database.execute( + "UPDATE \"LiteLLM_SpendLogDelivery\" SET state='applied', applied_at=CURRENT_TIMESTAMP - INTERVAL '400 days'" + ) + repository = AccountingDeliveryRepository(delivery_client) + assert await repository.cleanup_applied() == 1 + assert await repository.cleanup_applied() == 0 + assert delivery_database.execute('SELECT request_id FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("batch",)] + metrics = await repository.metrics() + assert metrics.batch_count == 1 + assert metrics.table_bytes > 0 + + +@pytest.mark.parametrize("disabled", [False, True]) +@pytest.mark.asyncio +async def test_ledger_batch_retrieves_charge_once_in_every_logging_mode( + delivery_database, delivery_client, monkeypatch, disabled +): + from types import SimpleNamespace + + from litellm.proxy import proxy_server + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.utils import jsonify_object + + completed = { + **_row("batch_completed"), "call_type": "aretrieve_batch", "status": "success", "spend": 0.5, + "metadata": '{"status":"success","private":"request detail"}', + "accounting_payload": { + "request_id": "batch_completed", "call_type": "aretrieve_batch", "status": "success", "spend": 0.5, + "metadata": {"accounting_savings": {"compression": 0.2}}, + }, + } + monkeypatch.setattr(proxy_server, "general_settings", {"spend_logs_metadata_fields": {"include": ["status"]}}) + client = SimpleNamespace(db=delivery_client, writer_db=delivery_client, jsonify_object=jsonify_object) + writer = DBSpendUpdateWriter() + first = await writer._claim_settle_once_cost_row(payload=completed, prisma_client=client, disable_spend_logs=disabled) + again = await writer._claim_settle_once_cost_row( + payload={**completed, "startTime": "2026-11-05T12:00:00+00:00"}, prisma_client=client, disable_spend_logs=disabled + ) + assert (first, again) == (True, False) + assert delivery_database.execute('SELECT spend, metadata FROM "LiteLLM_SpendLogs"').fetchall() == ( + [] if disabled else [(0.5, {"status": "success"})] + ) + assert delivery_database.execute( + 'SELECT delivery_id, accounting_payload FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("batch_completed", completed["accounting_payload"])] + + +@pytest.mark.parametrize("setting", ["retention_days", "batch_size"]) +@pytest.mark.asyncio +async def test_invalid_cleanup_settings_preserve_receipts(delivery_database, delivery_client, setting): + _insert(delivery_database, (_row("protected"),)) + with pytest.raises(ValueError, match="positive"): + await AccountingDeliveryRepository(delivery_client).cleanup_applied(**{setting: 0}) + assert delivery_database.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("protected", "pending") + ] + + +@pytest.mark.parametrize("setting", ["max_attempts", "batch_size"]) +@pytest.mark.asyncio +async def test_invalid_worker_settings_preserve_pending_work(delivery_database, delivery_client, setting): + from unittest.mock import AsyncMock + + _insert(delivery_database, (_row("protected"),)) + apply = AsyncMock() + with pytest.raises(ValueError, match="positive"): + await AccountingDeliveryRepository(delivery_client).apply_pending(apply, **{setting: 0}) + apply.assert_not_awaited() + assert delivery_database.execute( + 'SELECT request_id, state::text, attempts FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("protected", "pending", 0)] diff --git a/tests/integration/database/test_spendlogs_ledger_parity.py b/tests/integration/database/test_spendlogs_ledger_parity.py new file mode 100644 index 00000000000..e1894aa66bf --- /dev/null +++ b/tests/integration/database/test_spendlogs_ledger_parity.py @@ -0,0 +1,1437 @@ +import contextlib +import asyncio +import json +import os +import uuid +import subprocess +import sys +import time +from collections.abc import Iterator +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import psycopg +from psycopg import sql +import pytest +import litellm + +from litellm.accounting.application import apply_deliveries +from litellm.caching.dual_cache import DualCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.proxy.spend_tracking.spend_tracking_utils import spend_log_row_with_retained_metadata +from litellm.proxy import proxy_server +from litellm.proxy._types import SpendLogsMetadataFields +from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.model_usage_rollup import flush_model_usage_transactions +from litellm.proxy.guardrails.usage_tracking import PendingRollups, process_spend_logs_guardrail_usage +from litellm.proxy.utils import PrismaClient +from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows +from litellm.proxy.hooks.model_max_budget_limiter import PROXY_VirtualKeyModelMaxBudgetLimiter +from litellm.repositories.accounting_delivery_repository import AccountingDeliveryRepository +from litellm.repositories.table_repositories import SpendLogsRepository +from litellm.responses.streaming_iterator import drain_logging_tasks +from litellm.types.utils import ModelResponse, Usage +from tests.integration._support.database import scratch_database + +_ROOT: Final = Path(__file__).resolve().parents[3] +_NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) +_ENTITY_TABLES: Final = ( + "LiteLLM_VerificationToken", + "LiteLLM_UserTable", + "LiteLLM_TeamTable", + "LiteLLM_TeamMembership", + "LiteLLM_OrganizationTable", + "LiteLLM_OrganizationMembership", + "LiteLLM_EndUserTable", + "LiteLLM_AgentsTable", + "LiteLLM_TagTable", + "LiteLLM_ProjectTable", + "LiteLLM_ModelAccessGroupBudgetTable", +) +_DAILY_TABLES: Final = ( + "LiteLLM_DailyUserSpend", + "LiteLLM_DailyTeamSpend", + "LiteLLM_DailyOrganizationSpend", + "LiteLLM_DailyEndUserSpend", + "LiteLLM_DailyAgentSpend", + "LiteLLM_DailyTagSpend", + "LiteLLM_DailyModelUsage", + "LiteLLM_DailyGuardrailUsageUnits", + "LiteLLM_BudgetWindowSpend", +) + + +@pytest.mark.parametrize("batch_size,row_count,expected,distinct_models", [(None, 251, 251, False), ("200", 251, 200, False), (None, 10001, 10000, False), (None, 3001, 3001, True)]) +@pytest.mark.asyncio +async def test_worker_applies_the_configured_backlog_in_one_transaction( + parity_database: str, monkeypatch: pytest.MonkeyPatch, batch_size: str | None, row_count: int, expected: int, distinct_models: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + if batch_size is None: + monkeypatch.delenv("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", raising=False) + else: + monkeypatch.setenv("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", batch_size) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + writer = DBSpendUpdateWriter() + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + stored = await client.writer_db.query_raw('SELECT to_jsonb(s) AS row FROM "LiteLLM_SpendLogs" s') + delivery = (await client.writer_db.query_raw('SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"'))[0]["accounting_payload"] + assert await SpendLogsRepository(client).insert_with_deliveries(tuple( + {**stored[0]["row"], "request_id": f"backlog-{index:05}", "model": f"backlog-model-{index}" if distinct_models else stored[0]["row"]["model"], "accounting_payload": {**delivery, "request_id": f"backlog-{index:05}", "model": f"backlog-model-{index}" if distinct_models else delivery["model"]}} for index in range(row_count - 1) + )) == row_count - 1 + started = time.monotonic() + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + print(f"Ledger batch applied={expected} queued={row_count} seconds={time.monotonic() - started:.3f}") + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery" WHERE state=\'applied\'').fetchone() == (expected,) + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery" WHERE state=\'pending\'').fetchone() == (row_count - expected,) + assert connection.execute('SELECT COUNT(DISTINCT applied_at) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (1,) + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (expected * 0.25,) + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_DailyUserSpend"').fetchone() == (expected if distinct_models else 1,) + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("ledger", [False, True]) +@pytest.mark.parametrize("exclude_messages", [False, True]) +@pytest.mark.asyncio +async def test_registered_proxy_callbacks_charge_model_windows_before_persisting_spend( + parity_database: str, monkeypatch: pytest.MonkeyPatch, ledger: bool, exclude_messages: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.utils import Rules, function_setup + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "false") + for name in ("callbacks", "input_callback", "success_callback", "failure_callback", "_async_success_callback", "_async_failure_callback"): + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["messages"] if exclude_messages else []) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": ledger}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server.proxy_logging_obj, "db_spend_update_writer", writer) + limiter = PROXY_VirtualKeyModelMaxBudgetLimiter( + dual_cache=DualCache(), clock=lambda: _NOW.timestamp(), + shared_windows=DatabaseSharedWindows(prisma_client_source=lambda: client), + ) + monkeypatch.setattr(proxy_server, "model_max_budget_limiter", limiter) + litellm.logging_callback_manager.add_litellm_callback(limiter) + proxy_server.cost_tracking() + logging, _ = function_setup("acompletion", Rules(), _NOW, model="ledger-test", messages=[], litellm_call_id="registered-window") + logging.update_environment_variables( + optional_params={}, custom_llm_provider="openai", + litellm_params={"metadata": {"user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", "user_api_key_model_max_budget": { + "ledger-test": {"max_budget": 100, "budget_duration": "1d"} + }}}, + ) + logging.model_call_details["response_cost"] = 0.25 + response = ModelResponse(id="registered-window", model="ledger-test", usage=Usage(prompt_tokens=10, completion_tokens=5)) + await client.db.connect() + try: + await logging.async_success_handler(response, _NOW, _NOW, standard_logging_object={ + "model": "ledger-test", "model_group": "ledger-test", "response_cost": 0.25, + "metadata": {"user_api_key_hash": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"}, + "model_map_information": {}, "hidden_params": {}, "request_tags": [], + }) + await drain_logging_tasks() + await SpendLogsRepository(client).insert_with_deliveries(tuple(client.spend_log_transactions)) + if ledger: + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + await writer._commit_spend_updates_to_db_without_redis_buffer(client, 0, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_BudgetWindowSpend" WHERE entity_type=\'model_max_budget\'').fetchall() == [(0.25,)] + assert connection.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [(0.25,)] + finally: + await client.db.disconnect() + + +@pytest.fixture(scope="module") +def ledger_schema() -> str: + return subprocess.run( + [ + sys.executable, + "-I", + "-m", + "prisma", + "migrate", + "diff", + "--from-empty", + "--to-schema-datamodel", + str(_ROOT / "schema.prisma"), + "--script", + ], + check=True, + capture_output=True, + text=True, + cwd=_ROOT, + ).stdout + + +@pytest.fixture +def parity_database(ledger_schema: str) -> Iterator[str]: + with scratch_database("spendlogs_parity") as url, psycopg.connect(url, autocommit=True) as connection: + connection.execute(ledger_schema) + connection.execute( + "INSERT INTO \"LiteLLM_BudgetTable\" (budget_id, created_by, updated_by) VALUES ('budget', 'test', 'test')" + ) + connection.execute("INSERT INTO \"LiteLLM_UserTable\" (user_id) VALUES ('user')") + connection.execute("INSERT INTO \"LiteLLM_TeamTable\" (team_id) VALUES ('team')") + connection.execute( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by) " + "VALUES ('org', 'org', 'budget', 'test', 'test')" + ) + connection.execute("INSERT INTO \"LiteLLM_TeamMembership\" (team_id, user_id) VALUES ('team', 'user')") + connection.execute( + "INSERT INTO \"LiteLLM_OrganizationMembership\" (organization_id, user_id) VALUES ('org', 'user')" + ) + connection.execute("INSERT INTO \"LiteLLM_VerificationToken\" (token) VALUES ('2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683')") + connection.execute("INSERT INTO \"LiteLLM_EndUserTable\" (user_id) VALUES ('end-user')") + connection.execute( + 'INSERT INTO "LiteLLM_AgentsTable" ' + "(agent_id, agent_name, agent_card_params, created_by, updated_by) " + "VALUES ('agent', 'agent', '{}', 'test', 'test')" + ) + connection.execute( + 'INSERT INTO "LiteLLM_ProjectTable" (project_id, created_by, updated_by) ' + "VALUES ('project', 'test', 'test')" + ) + connection.execute("INSERT INTO \"LiteLLM_TagTable\" (tag_name) VALUES ('tag')") + connection.execute("INSERT INTO \"LiteLLM_ModelAccessGroupBudgetTable\" (access_group_name) VALUES ('group')") + yield url + + +def _snapshot(url: str) -> dict[str, list[object]]: + with psycopg.connect(url) as connection: + return { + table: [ + row[0] + for row in connection.execute( + sql.SQL( + "SELECT to_jsonb(t) - 'id' - 'created_at' - 'updated_at' - 'request_id' - 'last_active' FROM {} t " + "ORDER BY (to_jsonb(t) - 'id' - 'created_at' - 'updated_at' - 'request_id' - 'last_active')::text" + ).format(sql.Identifier(table)) + ).fetchall() + ] + for table in (*_ENTITY_TABLES, *_DAILY_TABLES) + } + + +def _reset(url: str) -> None: + with psycopg.connect(url, autocommit=True) as connection: + for table in _ENTITY_TABLES: + connection.execute(sql.SQL("UPDATE {} SET spend=0").format(sql.Identifier(table))) + for table in (*_DAILY_TABLES, "LiteLLM_SpendLogs", "LiteLLM_SpendLogDelivery"): + connection.execute(sql.SQL("TRUNCATE {}").format(sql.Identifier(table))) + connection.execute('UPDATE "LiteLLM_VerificationToken" SET total_spend=0') + connection.execute('UPDATE "LiteLLM_TeamMembership" SET total_spend=0') + + +async def _assert_legacy_database_parity( + parity_database: str, + monkeypatch: pytest.MonkeyPatch, + status: str, + origin: str | None, + cost: float, + retention: str, + reset_kind: str | None = None, + correction_order: str | None = None, + toggle_after_capture: bool = False, + fallback_narrowing: bool = False, + failing_table: str | None = None, + organization_id: str | None = "org", + export_url: str | None = None, + export_failure: bool = False, +): + monkeypatch.setitem( + litellm.model_cost, + "ledger-test", + { + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.004, + "cache_read_input_token_cost": 0.0005, + "cache_creation_input_token_cost": 0.003, + "litellm_provider": "openai", + "mode": "chat", + "max_tokens": 4096, + }, + ) + if fallback_narrowing: + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=[{ + "model_name": "ledger-test", + "litellm_params": {"model": "openai/ledger-test", "api_key": "test"}, + "model_info": {"id": "deployment", "access_groups": ["group"]}, + }])) + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + "INSERT INTO \"LiteLLM_ModelAccessGroupBudgetTable\" (access_group_name) VALUES ('unserved-group')" + ) + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + await client.db.connect() + results = [] + try: + for path in ("legacy", "ledger"): + _reset(parity_database) + if reset_kind is not None: + _prime_reset(parity_database, reset_kind) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": path == "ledger"}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", path == "ledger" and retention == "lean") + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server.proxy_logging_obj, "db_spend_update_writer", writer) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + monkeypatch.setattr(proxy_server, "spend_counter_cache", DualCache()) + for cache_key in ("2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", "team_id:team"): + await proxy_server.user_api_key_cache.async_set_cache( + key=cache_key, + value={ + "budget_limits": [ + {"budget_duration": "1d", "max_budget": 100, "reset_at": "2026-11-04T00:00:00+00:00"} + ] + }, + ) + request_kwargs = { + "model": "ledger-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "request", + "standard_logging_object": { + "model": "ledger-test", + "model_group": "ledger-test", + "metadata": { + "user_api_key_hash": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cache_read_input_tokens": 4, + "cache_creation_input_tokens": 1, + }, + }, + "model_map_information": {}, + "hidden_params": {}, + "request_tags": ["tag"], + "model_id": "deployment", + "response_cost": cost, + "autorouter_savings": 0.125, + "guardrail_information": [ + { + "guardrail_id": "guard", + "guardrail_name": "guard", + "guardrail_status": "success", + "guardrail_usage": {"text": 2}, + "guardrail_cost_by_unit": {"text": 0.02}, + "guardrail_cost_in_spend": True, + } + ], + }, + "agent_id": "agent", + "user": "end-user", + "litellm_params": { + "metadata": { + "user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + "user_api_key_model_max_budget": {"ledger-test": {"max_budget": 100, "budget_duration": "1d"}}, + "user_api_key_user_id": "user", + "user_api_key_team_id": "team", + "user_api_key_org_id": organization_id, + "user_api_key_project_id": "project", + "user_api_key_end_user_id": "end-user", + "tags": ["tag"], + "user_api_key_matched_model_access_groups": ["group", "unserved-group"] if fallback_narrowing else ["group"], + "model_info": {"id": "deployment"}, + "litellm_gateway_injected_cache": "deployment", + "compression_savings": {"tokens_before": 20, "tokens_after": 10, "tokens_saved": 10}, + "status": status, + **({BILLED_RETRY_ATTEMPT_METADATA_KEY: True} if origin == "billed_retry" else {}), + **({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin} if origin == "auto_router" else {}), + }, + "user": "end-user", + }, + } + limiter = PROXY_VirtualKeyModelMaxBudgetLimiter( + dual_cache=proxy_server.spend_counter_cache, + clock=lambda: _NOW.timestamp(), + shared_windows=DatabaseSharedWindows(prisma_client_source=lambda: client), + ) + if status == "success": + await limiter.async_log_success_event(request_kwargs, None, _NOW, _NOW) + charged = await writer.update_database( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + user_id="user", + end_user_id="end-user", + team_id="team", + org_id=organization_id, + project_id="project", + kwargs=request_kwargs, + completion_response=ModelResponse( + id="request", model="ledger-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + response_cost=cost, + ) + assert charged + await proxy_server.increment_spend_counters( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + team_id="team", + user_id="user", + response_cost=cost, + request_started_at=_NOW, + **({"windows_from_ledger": True} if path == "ledger" else {}), + ) + await drain_logging_tasks() + rows = tuple(client.spend_log_transactions) + client.spend_log_transactions = [] + assert len(rows) == 1 + if path == "ledger" and retention == "lean": + assert rows[0]["accounting_log_disabled"] is True + frozen = rows[0]["accounting_payload"] + if isinstance(frozen, str): + frozen = json.loads(frozen) + assert not {"messages", "response", "proxy_server_request"}.intersection(frozen) + stored_rows = ( + tuple(spend_log_row_with_retained_metadata(row, SpendLogsMetadataFields(include=())) for row in rows) + if retention == "stripped" + else rows + ) + if toggle_after_capture: + for enabled in (True, False, True): + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": enabled}) + await SpendLogsRepository(client).insert_with_deliveries(stored_rows) + elif path == "ledger" and export_url is not None: + import httpx + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.setenv("SPEND_LOGS_URL", export_url) + handler = AsyncHTTPHandler(timeout=10) + wire_rows = [ + {key: value.isoformat() if isinstance(value, datetime) else value for key, value in row.items()} + for row in stored_rows + ] + try: + if export_failure: + with pytest.raises(httpx.HTTPStatusError): + await ProxyUpdateSpend.update_spend_logs(0, client, handler, proxy_server.proxy_logging_obj, wire_rows) + else: + await ProxyUpdateSpend.update_spend_logs(0, client, handler, proxy_server.proxy_logging_obj, wire_rows) + finally: + await handler.close() + else: + await SpendLogsRepository(client).insert_with_deliveries(stored_rows) + if path == "ledger" and (retention == "lean" or export_url is not None): + assert await client.writer_db.query_raw('SELECT COUNT(*) AS count FROM "LiteLLM_SpendLogs"') == [{"count": 0}] + if reset_kind is not None: + await _apply_reset(reset_kind, client) + if correction_order == "before": + await _publish_savings_correction(client, stored_rows[0]) + if path != "ledger": + await writer._commit_spend_updates_to_db_without_redis_buffer(client, 0, proxy_server.proxy_logging_obj) + await writer.commit_daily_tag_spend_to_db(client, 0, proxy_server.proxy_logging_obj) + await flush_model_usage_transactions(prisma_client=client, transactions=client.model_usage_transactions) + client.model_usage_transactions = [] + await process_spend_logs_guardrail_usage(client, rows, pending=PendingRollups()) + else: + + async def apply(transaction, deliveries, writer: DBSpendUpdateWriter = writer): + await apply_deliveries( + transaction, + deliveries, + prisma_client=client, + writer=writer, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + + if failing_table is not None: + before_failure = _snapshot(parity_database) + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute(sql.SQL('ALTER TABLE {} ADD CONSTRAINT reject_spend CHECK (spend = 0)').format(sql.Identifier(failing_table))) + failed = await AccountingDeliveryRepository(client.writer_db).apply_pending(apply) + assert (failed.applied, failed.failed) == (0, 1) + assert _snapshot(parity_database) == before_failure + with psycopg.connect(parity_database, autocommit=True) as connection: + assert connection.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("pending", 1)] + connection.execute(sql.SQL('ALTER TABLE {} DROP CONSTRAINT reject_spend').format(sql.Identifier(failing_table))) + connection.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at=CURRENT_TIMESTAMP') + result = await AccountingDeliveryRepository(client.writer_db).apply_pending(apply) + assert result.applied == 1, result + await writer._commit_spend_updates_to_db_without_redis_buffer(client, 0, proxy_server.proxy_logging_obj) + if correction_order == "after": + await _publish_savings_correction(client, stored_rows[0]) + results.append(_snapshot(parity_database)) + differences = { + (index, table): (results[0][table], result[table]) + for index, result in enumerate(results[1:], 1) + for table in results[0] + if results[0][table] != result[table] + } + assert not differences, differences + prior_spend = 5 if reset_kind is not None and reset_kind != "scheduled" else 0 + assert results[-1]["LiteLLM_UserTable"][0]["spend"] == cost + prior_spend + daily = results[-1]["LiteLLM_DailyUserSpend"][0] + assert daily["compression_savings_spend"] > 0, results[-1]["LiteLLM_DailyUserSpend"] + assert daily["prompt_caching_savings_spend"] != 0 + assert daily["gateway_injected_caching_savings_spend"] != 0 + assert daily["autorouter_savings_spend"] == pytest.approx( + (0 if origin == "auto_router" else 0.125) + (0.2 if correction_order else 0) + ) + group_spend = {row["access_group_name"]: row["spend"] for row in results[-1]["LiteLLM_ModelAccessGroupBudgetTable"]} + assert group_spend == {"group": cost + prior_spend, **({"unserved-group": prior_spend} if fallback_narrowing else {})} + assert results[-1]["LiteLLM_DailyGuardrailUsageUnits"] + windows = results[-1]["LiteLLM_BudgetWindowSpend"] + assert len(windows) == (2 if cost else 0) + (1 if status == "success" else 0) + assert all(row["spend"] == cost for row in windows) + if reset_kind == "window_rollover": + model_window = next(row for row in windows if row["entity_type"] == "model_max_budget") + assert model_window["window_start"].startswith("2026-11-04") + if reset_kind in ("key", "global", "scheduled"): + assert results[-1]["LiteLLM_VerificationToken"][0]["spend"] == cost + if reset_kind in ("global", "scheduled"): + assert results[-1]["LiteLLM_TeamTable"][0]["spend"] == cost + if reset_kind == "team_member": + assert results[-1]["LiteLLM_TeamMembership"][0]["spend"] == cost + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize( + "status,origin,cost", + [ + ("success", None, 0.25), + ("failure", None, 0.25), + ("success", None, 0.0), + ("success", "auto_router", 0.25), + ("failure", "billed_retry", 0.25), + ], +) +@pytest.mark.parametrize("retention", ["full", "stripped", "lean"]) +@pytest.mark.asyncio +async def test_stored_ledger_matches_real_legacy_database_updates( + parity_database: str, monkeypatch: pytest.MonkeyPatch, status: str, origin: str | None, cost: float, retention: str +): + await _assert_legacy_database_parity(parity_database, monkeypatch, status, origin, cost, retention) + + +@pytest.mark.parametrize("retention", ["full", "stripped", "lean"]) +@pytest.mark.asyncio +async def test_missing_organization_matches_legacy_daily_rollups(parity_database, monkeypatch, retention): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, retention, organization_id=None + ) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT count(*) FROM "LiteLLM_DailyOrganizationSpend"').fetchone() == (0,) + + +@pytest.mark.parametrize("reset_kind", ["scheduled", "key", "team_member", "global", "window_rollover"]) +@pytest.mark.asyncio +async def test_pending_delivery_after_reset_matches_queued_legacy_spend( + parity_database: str, monkeypatch: pytest.MonkeyPatch, reset_kind: str +): + await _assert_legacy_database_parity(parity_database, monkeypatch, "success", None, 0.25, "full", reset_kind) + + +def _prime_reset(url: str, reset_kind: str) -> None: + with psycopg.connect(url, autocommit=True) as connection: + for table in _ENTITY_TABLES: + connection.execute(sql.SQL("UPDATE {} SET spend=5").format(sql.Identifier(table))) + if reset_kind == "scheduled": + for table in ("LiteLLM_VerificationToken", "LiteLLM_UserTable", "LiteLLM_TeamTable", "LiteLLM_BudgetTable"): + connection.execute( + sql.SQL("UPDATE {} SET budget_duration='1d', budget_reset_at='2026-11-01'").format( + sql.Identifier(table) + ) + ) + connection.execute("UPDATE \"LiteLLM_ModelAccessGroupBudgetTable\" SET budget_id='budget'") + + +async def _apply_reset(reset_kind: str, client: PrismaClient) -> None: + from litellm.proxy._types import ResetSpendRequest, UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob + from litellm.proxy.management_endpoints.key_management_endpoints import reset_key_spend_fn + from litellm.proxy.management_endpoints.team_endpoints import reset_team_member_spend_fn + from litellm.proxy.spend_tracking.spend_management_endpoints import global_spend_reset + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + if reset_kind == "window_rollover": + from litellm.proxy.db.budget_window_spend_writer import roll_window_spend_row + + windows = await client.writer_db.query_raw('SELECT * FROM "LiteLLM_BudgetWindowSpend"') + assert windows + for window in windows: + await roll_window_spend_row( + client, window["entity_type"], window["entity_id"], window["window_duration"], _NOW + timedelta(days=2) + ) + elif reset_kind == "global": + await global_spend_reset() + elif reset_kind == "key": + await reset_key_spend_fn( + key="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", data=ResetSpendRequest(reset_to=0), user_api_key_dict=admin, litellm_changed_by=None + ) + elif reset_kind == "team_member": + await reset_team_member_spend_fn( + team_id="team", user_id="user", data=ResetSpendRequest(reset_to=0), user_api_key_dict=admin + ) + else: + await ResetBudgetJob(proxy_server.proxy_logging_obj, client).reset_budget() + + +@pytest.mark.parametrize("correction_order", ["before", "after"]) +@pytest.mark.asyncio +async def test_baseline_correction_order_preserves_frozen_request_savings( + parity_database: str, monkeypatch: pytest.MonkeyPatch, correction_order: str +): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "full", correction_order=correction_order + ) + + +async def _publish_savings_correction(client: PrismaClient, row) -> None: + from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING + from litellm.proxy.db.baseline_accounting import ( + BaselinePublication, + DailyBaselineAttribution, + DailyBaselineTarget, + _Change, + _publish, + ) + + metadata_before = ( + await client.writer_db.query_raw( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', row["request_id"] + ) + )[0]["metadata"] + if isinstance(metadata_before, str): + metadata_before = json.loads(metadata_before) + change = _Change( + request_id=row["request_id"], + publication=BaselinePublication( + comparison_id="autorouter-baseline:v3:" + "a" * 64, + comparison_started_at=_NOW.timestamp(), + status="estimated", + reason="modeled", + provenance="modeled", + actual_spend=0.25, + baseline_spend=0.575, + ), + api_key=row["api_key"], + session_id="session", + router_name="router", + baseline_model="ledger-test", + covered_delta=0, + actual_delta=0.0, + savings_delta=0.2, + daily=DailyBaselineAttribution( + date=_NOW.date().isoformat(), + api_key=row["api_key"], + model=row["model"], + custom_llm_provider=row["custom_llm_provider"], + model_group=row["model_group"], + endpoint=ROUTE_ENDPOINT_MAPPING.get(row["call_type"]), + targets=tuple( + DailyBaselineTarget(entity=entity, entity_id=identity) + for entity, identity in ( + ("user", "user"), + ("team", "team"), + ("org", "org"), + ("end_user", "end-user"), + ("agent", "agent"), + ("tag", "tag"), + ) + ), + ), + ) + async with client.writer_db.tx() as transaction: + await _publish(transaction, (change,)) + metadata_after = ( + await client.writer_db.query_raw( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', row["request_id"] + ) + )[0]["metadata"] + if isinstance(metadata_after, str): + metadata_after = json.loads(metadata_after) + assert metadata_after["autorouter_savings"] == pytest.approx(0.325) + if "accounting_payload" in row: + delivery = (await client.writer_db.query_raw( + 'SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery" WHERE request_id=$1', row["request_id"] + ))[0]["accounting_payload"] + assert delivery["metadata"]["accounting_savings"]["autorouter"] == pytest.approx(0.125) + + +@pytest.mark.asyncio +async def test_flag_changes_do_not_reclassify_or_reapply_buffered_rows( + parity_database: str, + monkeypatch: pytest.MonkeyPatch, +): + await _assert_legacy_database_parity( + parity_database, + monkeypatch, + "success", + None, + 0.25, + "stripped", + toggle_after_capture=True, + ) + + +@pytest.mark.parametrize("entity_type", ["key", "team"]) +@pytest.mark.asyncio +async def test_window_seed_excludes_pending_and_parked_but_preserves_legacy_cutoff( + parity_database: str, + monkeypatch: pytest.MonkeyPatch, + entity_type: str, +): + from datetime import timedelta + from litellm.proxy.db.budget_window_spend_writer import spend_logs_seed_totals, commit_window_spend_updates + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + await client.db.connect() + try: + for request_id, cost, offset in ( + ("old-legacy", 2.0, -1), + ("current-legacy", 5.0, 0), + ("applied", 3.0, 1), + ("pending", 7.0, 0), + ("parked", 11.0, 0), + ): + await client.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_SpendLogs" ' + '(request_id,call_type,api_key,team_id,spend,"startTime","endTime") ' + "VALUES ($1,'acompletion','key','team',$2::float8,$3::timestamp,$3::timestamp)", + request_id, + cost, + _NOW + timedelta(seconds=offset), + ) + for request_id, state in (("applied", "applied"), ("pending", "pending"), ("parked", "parked")): + await client.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id,state,accounting_payload,is_batch) VALUES ($1,$2::"AccountingDeliveryState",$$ {} $$::jsonb,false)', + request_id, + state, + ) + start = _NOW - timedelta(hours=1) + totals = await spend_logs_seed_totals(client, entity_type, entity_type, start, _NOW) + assert totals is not None and totals.total == 10.0 and totals.before_batch == 5.0 + unbounded = await spend_logs_seed_totals(client, entity_type, entity_type, start, None) + assert unbounded is not None and unbounded.total == unbounded.before_batch == 10.0 + await commit_window_spend_updates( + client, + ( + { + "entity_type": entity_type, + "entity_id": entity_type, + "window_duration": "1d", + "window_start": start.isoformat(), + "started_at": _NOW.isoformat(), + "spend": 5.0, + }, + ), + ) + rows = await client.writer_db.query_raw('SELECT spend FROM "LiteLLM_BudgetWindowSpend"') + assert rows[0]["spend"] == 10.0 + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("disable_logs", [False, True]) +@pytest.mark.asyncio +async def test_batch_claim_failure_queues_an_atomic_retry_and_repeated_retrieve_does_not_charge( + parity_database: str, monkeypatch: pytest.MonkeyPatch, disable_logs: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", disable_logs) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + + + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + 'ALTER TABLE "LiteLLM_SpendLogDelivery" ADD CONSTRAINT reject_batch ' + "CHECK (request_id <> 'batch_batch_cost')" + ) + assert await _retrieve_batch(writer, 0.25) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,) + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,) + assert len(client.spend_log_transactions) == 1 + queued = tuple(client.spend_log_transactions) + assert "accounting_payload" in queued[0] + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery" DROP CONSTRAINT reject_batch') + assert await SpendLogsRepository(client).insert_with_deliveries(queued) == 1 + assert await _retrieve_batch(writer, 0.25) is False + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=\'2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683\'').fetchone() == (0.25,) + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0.25,) + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(not disable_logs),) + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_failed_claim_over_a_legacy_zero_batch_row_still_charges_it_once( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": False}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0) + await drain_logging_tasks() + await SpendLogsRepository(client).insert_with_deliveries(tuple(client.spend_log_transactions)) + client.spend_log_transactions = [] + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + with psycopg.connect(parity_database, autocommit=True) as connection: + assert connection.execute( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=\'batch_batch_cost\'' + ).fetchall() == [(0.0,)] + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery" RENAME TO "LiteLLM_SpendLogDelivery_unavailable"') + assert await _retrieve_batch(writer, 0.25) + queued = tuple(client.spend_log_transactions) + assert ["accounting_payload" in row for row in queued] == [True] + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery_unavailable" RENAME TO "LiteLLM_SpendLogDelivery"') + assert await SpendLogsRepository(client).insert_with_deliveries(queued) == 1 + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert await _retrieve_batch(writer, 0.25) is False + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=\'2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683\'').fetchone() == (0.25,) + assert connection.execute( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=\'batch_batch_cost\'' + ).fetchall() == [(0.0,)] + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_worker_exposes_invalid_accounting_as_delayed_then_parked_work( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from prometheus_client import REGISTRY + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + await client.db.connect() + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, created_at, accounting_payload, is_batch) ' + "VALUES ('invalid-accounting', CURRENT_TIMESTAMP - INTERVAL '1 hour', " + '$$ {"metadata": {"accounting_windows": [{}]}} $$::jsonb, false)' + ) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert REGISTRY.get_sample_value("litellm_accounting_pending") == 1 + assert REGISTRY.get_sample_value("litellm_accounting_parked") == 0 + with psycopg.connect(parity_database, autocommit=True) as connection: + assert connection.execute( + 'SELECT state::text, attempts, last_error FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("pending", 1, "KeyError")] + connection.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" SET attempts=4, next_attempt_at=CURRENT_TIMESTAMP' + ) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert REGISTRY.get_sample_value("litellm_accounting_pending") == 0 + assert REGISTRY.get_sample_value("litellm_accounting_parked") == 1 + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT state::text, attempts, last_error FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("parked", 5, "KeyError")] + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0,) + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_fallback_charges_only_groups_declared_by_the_served_deployment( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "stripped", fallback_narrowing=True + ) + + +@pytest.mark.asyncio +async def test_worker_skips_another_leader_and_releases_its_redis_lock_after_retry( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.caching.redis_cache import RedisCache + from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager + + monkeypatch.setenv("DATABASE_URL", parity_database) + cache = RedisCache(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), namespace=uuid.uuid4().hex) + other_leader = PodLockManager(cache) + writer = DBSpendUpdateWriter(redis_cache=cache) + writer.pod_lock_manager.redis_cache = cache + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + await client.db.connect() + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch) VALUES ($$invalid-row$$, $$ {"metadata": {"accounting_windows": [{}]}} $$::jsonb, false)') + assert await other_leader.acquire_lock(cronjob_id="accounting_delivery", allow_reentrant=False) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [(0,)] + await other_leader.release_lock(cronjob_id="accounting_delivery") + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [(1,)] + assert await other_leader.acquire_lock(cronjob_id="accounting_delivery", allow_reentrant=False) + finally: + await other_leader.release_lock(cronjob_id="accounting_delivery") + await client.db.disconnect() + await cache.disconnect() + + +async def _retrieve_batch(writer: DBSpendUpdateWriter, cost: float) -> bool: + return await writer.update_database( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", user_id="user", end_user_id=None, team_id="team", org_id=None, + kwargs={ + "model": "ledger-test", "call_type": "aretrieve_batch", "litellm_call_id": "batch", + "litellm_params": {"metadata": {"status": "success", "user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"}}, + }, + completion_response={"id": "batch", "status": "completed"}, + start_time=_NOW, end_time=_NOW, response_cost=cost, + ) + + +@pytest.mark.parametrize("ledger_after_zero_row,delivery_unavailable", [(False, False), (True, False), (True, True)]) +@pytest.mark.asyncio +async def test_zero_cost_legacy_batch_row_is_charged_once_across_a_flag_change( + parity_database: str, monkeypatch: pytest.MonkeyPatch, ledger_after_zero_row: bool, delivery_unavailable: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": False}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0) + await drain_logging_tasks() + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": ledger_after_zero_row}) + assert await _retrieve_batch(writer, 0.25) + await drain_logging_tasks() + assert await _retrieve_batch(writer, 0.25) is False + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0,) + assert connection.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [ + (0.0 if ledger_after_zero_row else 0.25,) + ] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == ( + int(ledger_after_zero_row), + ) + if delivery_unavailable: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery" RENAME TO temporarily_unavailable_delivery') + try: + await writer.db_update_spend_transaction_handler(client, 0, proxy_server.proxy_logging_obj) + finally: + if delivery_unavailable: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE temporarily_unavailable_delivery RENAME TO "LiteLLM_SpendLogDelivery"') + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0.25,) + assert connection.execute('SELECT spend, api_requests FROM "LiteLLM_DailyUserSpend"').fetchall() == [(0.25, 2)] + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("failing_table", [ + "LiteLLM_UserTable", "LiteLLM_VerificationToken", "LiteLLM_TeamTable", "LiteLLM_TeamMembership", + "LiteLLM_OrganizationMembership", "LiteLLM_OrganizationTable", "LiteLLM_ProjectTable", "LiteLLM_AgentsTable", "LiteLLM_EndUserTable", +]) +@pytest.mark.asyncio +async def test_destination_failure_rolls_back_all_earlier_updates_before_retry( + parity_database: str, monkeypatch: pytest.MonkeyPatch, failing_table: str +): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "full", failing_table=failing_table + ) + + +@pytest.mark.asyncio +async def test_spend_logs_cleanup_does_not_remove_pending_accounting(parity_database, monkeypatch): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + cleanup = SpendLogCleanup(clock=lambda: 0) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + delete_sql = 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id IN (SELECT request_id FROM "LiteLLM_SpendLogs" WHERE "startTime" < $1::timestamp LIMIT $2::integer)' + assert await cleanup._execute_delete_batch(client, delete_sql, _NOW + timedelta(days=1), "LiteLLM_SpendLogs", 60) == 1 + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert await _retrieve_batch(writer, 0.25) is False + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,) + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=$$user$$').fetchone() == (0.25,) + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("mode", ["local", "url", "disabled"]) +@pytest.mark.parametrize("receipt_state", ["pending", "parked", "applied", "retained"]) +@pytest.mark.asyncio +async def test_ledger_charged_batch_is_not_charged_again_after_flag_disable( + parity_database: str, monkeypatch: pytest.MonkeyPatch, receipt_state: str, mode: str +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", mode == "disabled") + if mode == "url": + monkeypatch.setenv("SPEND_LOGS_URL", "http://127.0.0.1:1") + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0) + assert await _retrieve_batch(writer, 0.25) + if receipt_state in ("applied", "retained"): + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database, autocommit=True) as connection: + if receipt_state == "retained": + connection.execute("UPDATE \"LiteLLM_SpendLogDelivery\" SET applied_at = CURRENT_TIMESTAMP - INTERVAL '400 days'") + if receipt_state == "parked": + connection.execute('UPDATE "LiteLLM_SpendLogDelivery" SET state=\'parked\'') + assert await AccountingDeliveryRepository(client.writer_db).cleanup_applied() == 0 + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": False}) + assert await _retrieve_batch(writer, 0.25) is False + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == ( + [(0.25,)] if mode != "disabled" else [] + ) + expected_spend = 0 if receipt_state == "parked" else 0.25 + expected_state = "parked" if receipt_state == "parked" else "applied" + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (expected_spend,) + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [(expected_state,)] + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("disabled", [False, True]) +@pytest.mark.parametrize("export_status", [200, 500]) +@pytest.mark.asyncio +async def test_export_failure_does_not_change_committed_accounting(parity_database, monkeypatch, export_status, disabled): + requests = asyncio.Queue() + + async def receive(reader, writer): + header = await reader.readuntil(b"\r\n\r\n") + length = next(int(line.split(b":", 1)[1]) for line in header.split(b"\r\n") if line.lower().startswith(b"content-length:")) + body = await reader.readexactly(length) + await requests.put(json.loads(body)) + writer.write(f"HTTP/1.1 {export_status} Result\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{{}}".encode()) + await writer.drain() + writer.close() + await writer.wait_closed() + + server = await asyncio.start_server(receive, "127.0.0.1", 0) + try: + port = server.sockets[0].getsockname()[1] + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "lean" if disabled else "full", + export_url=f"http://127.0.0.1:{port}", export_failure=export_status == 500 and not disabled, + ) + assert requests.qsize() == int(not disabled) + if disabled: + return + exported = await asyncio.wait_for(requests.get(), timeout=5) + assert len(exported) == 1 + assert exported[0]["spend"] == 0.25 + assert "accounting_payload" not in exported[0] + assert "accounting_log_disabled" not in exported[0] + finally: + server.close() + await server.wait_closed() + + +@pytest.mark.asyncio +async def test_spend_logs_url_without_an_export_client_keeps_local_logs_and_charges_once( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("SPEND_LOGS_URL", "http://127.0.0.1:1") + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + try: + assert await writer.update_database( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", user_id="user", end_user_id=None, team_id="team", org_id=None, + kwargs={ + "model": "ledger-test", "call_type": "acompletion", "litellm_call_id": "url-request", + "litellm_params": {"metadata": {"status": "success", "user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"}}, + }, + completion_response=ModelResponse( + id="url-request", model="ledger-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, end_time=_NOW, response_cost=0.25, + ) + await drain_logging_tasks() + rows = [ + {key: value.isoformat() if isinstance(value, datetime) else value for key, value in row.items()} + for row in client.spend_log_transactions + ] + client.spend_log_transactions = [] + await ProxyUpdateSpend.update_spend_logs(0, client, None, proxy_server.proxy_logging_obj, rows) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=\'url-request\'' + ).fetchall() == [(0.25,)] + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + assert connection.execute('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=\'2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683\'').fetchone() == (0.25,) + finally: + await client.db.disconnect() + + +@contextlib.asynccontextmanager +async def _ledger_client(parity_database: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + try: + yield client + finally: + await client.db.disconnect() + + +async def _record_call( + writer: DBSpendUpdateWriter, call_id: str, *, token: str = "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", user: str = "user", tags: object = None +) -> None: + metadata = {"status": "success", "user_api_key": token, **({"tags": tags} if tags is not None else {})} + assert await writer.update_database( + token=token, user_id=user, end_user_id=None, team_id="team", org_id=None, + kwargs={ + "model": "ledger-test", "call_type": "aembedding", "litellm_call_id": call_id, + "litellm_params": {"metadata": metadata}, + }, + completion_response=None, start_time=_NOW, end_time=_NOW, response_cost=0.25, + ) + await drain_logging_tasks() + + +def _wire(rows) -> list[dict[str, object]]: + return [ + {key: value.isoformat() if isinstance(value, datetime) else value for key, value in row.items()} + for row in rows + ] + + +async def _flush(writer: DBSpendUpdateWriter, client: PrismaClient) -> None: + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.utils import ProxyUpdateSpend + + rows = _wire(client.spend_log_transactions) + client.spend_log_transactions = [] + await ProxyUpdateSpend.update_spend_logs(0, client, None, proxy_server.proxy_logging_obj, rows) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + + +def _key_spend(parity_database: str) -> object: + with psycopg.connect(parity_database) as connection: + return connection.execute( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', ("2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683",) + ).fetchone() + + +@pytest.mark.asyncio +async def test_requests_sharing_a_client_call_id_are_each_charged_but_a_replay_is_not( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "shared-call-id") + await _record_call(writer, "shared-call-id") + buffered = _wire(client.spend_log_transactions) + await _flush(writer, client) + assert await SpendLogsRepository(client).insert_with_deliveries(buffered) == 0 + await _flush(writer, client) + assert _key_spend(parity_database) == (0.5,) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("shared-call-id", "applied"), ("shared-call-id", "applied")] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (1,) + + +@pytest.mark.asyncio +async def test_an_opaque_bearer_token_is_never_stored_in_a_delivery( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "opaque-call", token="opaque-bearer-secret") + await _flush(writer, client) + with psycopg.connect(parity_database) as connection: + stored = connection.execute('SELECT accounting_payload::text, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() + assert [state for _, state in stored] == ["applied"] + assert "opaque-bearer-secret" not in stored[0][0] + + +@pytest.mark.asyncio +async def test_an_evicted_spend_log_still_charges_its_request(parity_database: str, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.utils import enqueue_spend_logs + + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "evicted-call") + await _record_call(writer, "kept-call") + queued = tuple(client.spend_log_transactions) + client.spend_log_transactions = [] + await enqueue_spend_logs(client, queued, max_bytes=1) + assert [row.get("accounting_log_disabled") for row in client.spend_log_transactions] == [True, False] + await _flush(writer, client) + assert _key_spend(parity_database) == (0.5,) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT request_id FROM "LiteLLM_SpendLogs"').fetchall() == [("kept-call",)] + + +@pytest.mark.parametrize("isolation_budget", [5, 0], ids=["isolated", "budget_exhausted"]) +@pytest.mark.asyncio +async def test_a_spend_log_postgres_rejects_still_charges_its_request_and_the_rest_of_the_batch( + parity_database: str, monkeypatch: pytest.MonkeyPatch, isolation_budget: int +): + from prisma.errors import DataError + + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + class RejectsOneLog: + def __init__(self, repository: SpendLogsRepository) -> None: + self.repository = repository + + async def insert_with_deliveries(self, rows, *, write_spend_logs: bool = True) -> int: + if any(row["request_id"] == "poison-call" and not row.get("accounting_log_disabled") for row in rows): + raise DataError({"user_facing_error": {"message": 'invalid byte sequence for encoding "UTF8": 0x00'}}) + return await self.repository.insert_with_deliveries(rows, write_spend_logs=write_spend_logs) + + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "poison-call") + await _record_call(writer, "clean-call") + rows = tuple(client.jsonify_object(row) for row in client.spend_log_transactions) + client.spend_log_transactions = [] + await _create_spend_logs_with_poison_isolation(RejectsOneLog(SpendLogsRepository(client)), rows, isolation_budget) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert _key_spend(parity_database) == (0.5,) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT request_id FROM "LiteLLM_SpendLogs"').fetchall() == ( + [("clean-call",)] if isolation_budget else [] + ) + assert connection.execute( + 'SELECT state::text, COUNT(*) FROM "LiteLLM_SpendLogDelivery" GROUP BY 1' + ).fetchall() == [("applied", 2)] + + +@pytest.mark.asyncio +async def test_a_request_with_a_non_text_tag_is_still_charged(parity_database: str, monkeypatch: pytest.MonkeyPatch): + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "tagged-call", tags=[None, "real-tag"]) + await _flush(writer, client) + assert _key_spend(parity_database) == (0.25,) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT tag, spend FROM "LiteLLM_DailyTagSpend" WHERE tag=\'real-tag\'' + ).fetchall() == [("real-tag", 0.25)] + + +@pytest.mark.parametrize("threshold", ["count", "bytes"]) +@pytest.mark.asyncio +async def test_retained_batch_storage_threshold_alerts_without_deleting_receipts( + parity_database: str, monkeypatch: pytest.MonkeyPatch, threshold: str +): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.types.integrations.slack_alerting import AlertType + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("LITELLM_ACCOUNTING_BATCH_DELIVERY_ALERT_COUNT", "1" if threshold == "count" else "1000000") + monkeypatch.setenv("LITELLM_ACCOUNTING_DELIVERY_ALERT_BYTES", "1" if threshold == "bytes" else "1073741824") + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + alerts = SimpleNamespace(alerting_handler=AsyncMock()) + await client.db.connect() + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, state, applied_at, accounting_payload, is_batch) ' + "VALUES ('old-batch', 'applied', CURRENT_TIMESTAMP - INTERVAL '400 days', '{}', true)" + ) + await flush_accounting_deliveries(DBSpendUpdateWriter(), client, alerts) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("old-batch", "applied") + ] + table_bytes = connection.execute( + 'SELECT pg_total_relation_size(\'"LiteLLM_SpendLogDelivery"\')' + ).fetchone()[0] + alerts.alerting_handler.assert_awaited_once_with( + message=f"Accounting delivery storage requires review. Retained batches=1, table bytes={table_bytes}.", + level="High", alert_type=AlertType.failed_tracking_spend, + ) + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("backlog", ["retries", "expired"]) +@pytest.mark.parametrize("batch_size", [1, 1100]) +@pytest.mark.asyncio +async def test_worker_drains_recovered_and_expired_work_at_its_configured_capacity( + parity_database, monkeypatch, backlog, batch_size +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", str(batch_size)) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + with psycopg.connect(parity_database, autocommit=True) as connection: + if backlog == "retries": + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, attempts) ' + "SELECT 'retry-' || i, accounting_payload, false, 1 " + 'FROM "LiteLLM_SpendLogDelivery" CROSS JOIN generate_series(1, 3) i' + ) + else: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, state, applied_at) ' + "SELECT 'expired-' || i, '{}', false, 'applied', CURRENT_TIMESTAMP - INTERVAL '2 days' " + 'FROM generate_series(1, 1100) i' + ) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == ( + 0.25 * (1 + min(batch_size, 3)) if backlog == "retries" else 0.25, + ) + assert connection.execute( + 'SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery" WHERE state=\'pending\' OR request_id LIKE \'expired-%\'' + ).fetchone() == ((3 - min(batch_size, 3)) if backlog == "retries" else max(0, 1100 - batch_size * 2),) + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_analytics_overflow_across_requests_still_applies_every_charge( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + writer = DBSpendUpdateWriter() + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + stored = await client.writer_db.query_raw('SELECT to_jsonb(s) AS row FROM "LiteLLM_SpendLogs" s') + delivery = (await client.writer_db.query_raw('SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"'))[0][ + "accounting_payload" + ] + oversized_savings = {**delivery["metadata"], "compression_savings": {"tokens_saved": 2**62}} + assert await SpendLogsRepository(client).insert_with_deliveries( + tuple( + { + **stored[0]["row"], + "request_id": f"overflow-{index}", + "accounting_payload": {**delivery, "request_id": f"overflow-{index}", "metadata": oversized_savings}, + } + for index in range(3) + ) + ) == 3 + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT state::text, COUNT(*) FROM "LiteLLM_SpendLogDelivery" GROUP BY state' + ).fetchall() == [("applied", 4)] + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (1.0,) + assert connection.execute( + 'SELECT api_requests, compression_saved_tokens FROM "LiteLLM_DailyUserSpend"' + ).fetchone() == (4, 3 * 2**40) + finally: + await client.db.disconnect() diff --git a/tests/integration/management/test_skill_download_history.py b/tests/integration/management/test_skill_download_history.py new file mode 100644 index 00000000000..00445dd7e27 --- /dev/null +++ b/tests/integration/management/test_skill_download_history.py @@ -0,0 +1,163 @@ +import hashlib +import io +import os +import uuid +import zipfile +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from functools import partial +from pathlib import Path +from typing import Final + +import psycopg +import pytest +from pydantic import TypeAdapter + +from litellm.models.plugin_upload import PluginDownloadsResponse +from litellm.types.proxy.skill_hub import SkillHubList +from tests.integration._support.actors import GatewayClient, owned_settings_client +from tests.integration._support.client import Gateway, Scenario, object_value, string_value +from tests.integration._support.database import write_rows + +PLUGINS: Final = "/claude-code/plugins" +SKILLS: Final = "/skill-hub/skills" +HISTORY: Final = TypeAdapter(PluginDownloadsResponse) + + +@pytest.fixture(scope="module") +def gateway(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("skill-downloads") + with owned_settings_client(GatewayClient(session_gateway, {}), directory, {"public_skills_index": True}) as client: + with client.gateway.scenario() as publisher: + yield replace(client.gateway, key=publisher.key(user_id=publisher.user(user_role="proxy_admin"))) + + +def _delete(gateway: Gateway, name: str) -> None: + assert gateway.request("DELETE", f"{PLUGINS}/{name}").status_code in {200, 404} + + +def _upload(scenario: Scenario, name: str) -> str: + scenario.cleanups.callback(_delete, scenario.gateway, name) + content: Final = io.BytesIO() + with zipfile.ZipFile(content, "w") as archive: + archive.writestr("SKILL.md", f"---\nname: {name}\ndescription: Synthetic download regression\n---\n") + response: Final = scenario.gateway.request_multipart( + f"{SKILLS}/upload", {"version": "1.2.3"}, {"file": ("skill.zip", content.getvalue(), "application/zip")} + ) + assert response.status_code == 200, response.text + return string_value(object_value(scenario.gateway.get(f"{PLUGINS}/{name}")["source"])["url"]) + + +def _download(gateway: Gateway, key: str | None, url: str) -> str: + response: Final = gateway.client.get( + url, headers={"Authorization": f"Bearer {key}"} if key else {}, params={"user_id": "forged-user"} + ) + assert response.status_code == 200, response.text + return hashlib.sha256(response.content).hexdigest() + + +def _history(gateway: Gateway, name: str, **params: str) -> PluginDownloadsResponse: + history: Final = gateway.get(f"{SKILLS}/{name}/downloads", params) + assert gateway.get(f"{PLUGINS}/{name}/downloads", params) == history + return HISTORY.validate_python(history) + + +def test_attribution_survives_actor_deletion_and_name_reuse_starts_empty(gateway: Gateway) -> None: + with gateway.scenario() as skills: + name: Final = f"download-{uuid.uuid4().hex}" + legacy: Final = _upload(skills, name) + portable: Final = f"{SKILLS}/{name}/archive" + original_id: Final = string_value(gateway.get(f"{SKILLS}/{name}")["id"]) + assert gateway.client.get(f"{SKILLS}/{name}/content").status_code == 200 + assert _history(gateway, name).total_downloads == 0 + assert SkillHubList.model_validate(gateway.get(SKILLS, {"query": name})).skills[0].download_count == 0 + with gateway.scenario() as actors: + team: Final = actors.team(team_alias=name) + email: Final = f"{name}@example.test" + user: Final = actors.user(user_role="internal_user", user_email=email) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": user}}) + key: Final = actors.key(user_id=user, team_id=team, key_alias=name) + digests: Final = tuple(_download(gateway, key, url) for url in (legacy, portable)) + for url in (legacy, portable): + _download(gateway, None, url) + assert gateway.request("GET", url, key="sk-invalid").status_code == 401 + recorded: Final = _history(gateway, name) + assert (recorded.total_downloads, recorded.anonymous_downloads) == (4, 2) + assert SkillHubList.model_validate(gateway.get(SKILLS, {"query": name})).skills[0].download_count == 4 + assert tuple((row.version, row.authenticated, row.user_id, row.user_email, row.key_alias, + row.team_id, row.team_alias) for row in recorded.downloads) == ( + ("1.2.3", True, user, email, name, team, name), + ) * 2 + assert tuple(row.digest for row in recorded.downloads) == digests[::-1] + for prefix in (PLUGINS, SKILLS): + assert gateway.request("GET", f"{prefix}/{name}/downloads", key=key).status_code == 403 + assert gateway.client.get(f"{prefix}/{name}/downloads").status_code == 401 + assert _history(gateway, name) == recorded + _delete(gateway, name) + assert gateway.rows('SELECT id FROM "LiteLLM_ClaudeCodePluginDownload" WHERE plugin_id = %s', (original_id,)) == [] + _upload(skills, name) + assert gateway.get(f"{SKILLS}/{name}")["id"] != original_id + assert _history(gateway, name) == PluginDownloadsResponse(0, 0, (), None) + + +def test_concurrent_counts_and_keyset_pages_preserve_scope_and_deleted_cursors(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + name: Final = f"download-{uuid.uuid4().hex}" + legacy: Final = _upload(scenario, name) + portable: Final = f"{SKILLS}/{name}/archive" + key: Final = scenario.key() + with ThreadPoolExecutor(max_workers=4) as pool: + tuple(pool.map(partial(_download, gateway, key), (legacy, portable) * 4)) + initial: Final = _history(gateway, name) + ordered: Final = tuple(row.id for row in initial.downloads) + assert initial.total_downloads == len(frozenset(ordered)) == 8 + assert ordered == tuple(sorted(ordered, key=int, reverse=True)) + timestamp: Final = gateway.get(f"{PLUGINS}/{name}")["updated_at"] + with ThreadPoolExecutor(max_workers=4) as pool: + tuple(pool.map(partial(_download, gateway, None), (legacy, portable) * 8)) + assert gateway.get(f"{PLUGINS}/{name}")["updated_at"] == timestamp + assert _history(gateway, name) == replace(initial, total_downloads=24, anonymous_downloads=16) + first: Final = _history(gateway, name, limit="3") + assert tuple(row.id for row in first.downloads) == ordered[:3] + assert first.next_cursor == ordered[2] + _download(gateway, key, portable) + other: Final = f"{name}-other" + _download(gateway, key, _upload(scenario, other)) + write_rows( + 'DELETE FROM "LiteLLM_ClaudeCodePluginDownload" WHERE id IN (%s::bigint, %s::bigint)', + (ordered[2], ordered[4]), database_url=gateway.database_url, + ) + assert first.next_cursor is not None + second: Final = _history(gateway, name, limit="3", cursor=first.next_cursor) + assert tuple(row.id for row in second.downloads) == (ordered[3], ordered[5], ordered[6]) + assert (second.total_downloads, second.anonymous_downloads) == (23, 16) + assert second.next_cursor is not None + last: Final = _history(gateway, name, limit="3", cursor=second.next_cursor) + assert tuple(row.id for row in last.downloads) == (ordered[7],) + assert last.next_cursor is None + assert _history(gateway, other, cursor=first.next_cursor) == PluginDownloadsResponse(1, 0, (), None) + catalog: Final = SkillHubList.model_validate(gateway.get(SKILLS, {"query": name})) + assert {skill.name: skill.download_count for skill in catalog.skills} == {name: 23, other: 1} + + +def test_repeat_migration_is_catalog_only_while_download_tables_are_locked(gateway: Gateway) -> None: + migration: Final = (Path(__file__).resolve().parents[3] / + "litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql" + ).read_bytes() + with gateway.scenario() as scenario: + name: Final = f"download-{uuid.uuid4().hex}" + url: Final = _upload(scenario, name) + for key in (None, gateway.key): + _download(gateway, key, url) + before: Final = _history(gateway, name) + database_url: Final = gateway.database_url or os.environ["DATABASE_URL"] + with psycopg.connect(database_url) as locker, psycopg.connect(database_url, autocommit=True) as migrator: + locker.execute( + 'LOCK TABLE "LiteLLM_ClaudeCodePluginTable", "LiteLLM_ClaudeCodePluginDownload" IN ACCESS EXCLUSIVE MODE' + ) + for _ in range(2): + with migrator.transaction(): + migrator.execute("SET LOCAL statement_timeout = '500ms'") + migrator.execute(migration) + assert _history(gateway, name) == before diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py index bae775164dd..9d0e31058e4 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py @@ -140,6 +140,16 @@ def _assert_internal_server_error(response: httpx.Response) -> None: assert "InternalServerError" in string_value(error["message"]), response.text +def _assert_openai_overloaded(response: httpx.Response) -> None: + assert response.status_code == 503, response.text + error: Final = _openai_error(response) + assert set(error) == {"message", "type", "param", "code"}, response.text + assert string_value(error["message"]), response.text + assert error["type"] == "service_unavailable_error", response.text + assert error["param"] is None, response.text + assert error["code"] == "server_is_overloaded", response.text + + def _one_post(wire: Wire, target: str) -> None: assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)] @@ -361,7 +371,7 @@ def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_rela ] -def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: Gateway) -> None: +def test_chat_completions_overloaded_frame_before_content_answers_503(gateway: Gateway) -> None: prompt: Final = _prompt() with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) @@ -370,24 +380,24 @@ def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: G "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0}, ) - _assert_internal_server_error(response) + _assert_openai_overloaded(response) _one_post(wire, "/v1/messages") -def test_chat_completions_529_answers_500_to_the_sync_sdk(gateway: Gateway) -> None: +def test_chat_completions_529_answers_503_to_the_sync_sdk(gateway: Gateway) -> None: with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) with pytest.raises(openai.APIStatusError) as caught: _openai_sdk(gateway).chat.completions.create( model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0} ) - assert caught.value.status_code == 500, caught.value.response.text - _assert_internal_server_error(caught.value.response) + assert caught.value.status_code == 503, caught.value.response.text + _assert_openai_overloaded(caught.value.response) _one_post(wire, "/v1/messages") - _assert_logged_failure(model, rows=1, error_class="InternalServerError", code="500") + _assert_logged_failure(model, rows=1, error_class="InternalServerError", code="503") -async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: Gateway) -> None: +async def test_chat_completions_stream_529_answers_503_to_the_async_sdk(gateway: Gateway) -> None: with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) with pytest.raises(openai.APIStatusError) as caught: @@ -397,19 +407,31 @@ async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: stream=True, extra_body={"num_retries": 0}, ) - assert caught.value.status_code == 500, caught.value.response.text - _assert_internal_server_error(caught.value.response) + assert caught.value.status_code == 503, caught.value.response.text + _assert_openai_overloaded(caught.value.response) + _one_post(wire, "/v1/messages") + + +def test_completions_529_answers_503(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/completions", + {"model": model, "prompt": _prompt(), "max_tokens": 16, "num_retries": 0}, + ) + _assert_openai_overloaded(response) _one_post(wire, "/v1/messages") @pytest.mark.parametrize("shape", ("non-stream", "stream")) -def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None: +def test_responses_529_answers_503(gateway: Gateway, shape: Shape) -> None: with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) response: Final = gateway.request( "POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0} ) - _assert_internal_server_error(response) + _assert_openai_overloaded(response) _one_post(wire, "/v1/messages") diff --git a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py index 044c58e44cc..e86ed98fae7 100644 --- a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py +++ b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py @@ -215,7 +215,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]: ( pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"), pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"), - pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"), + pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 503, "500", "InternalServerError", id="chat-anthropic"), ), ) def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status( @@ -245,7 +245,7 @@ def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]: ("route", "model", "answered"), ( pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"), - pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat"), + pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat"), ), ) def test_the_server_span_carries_the_status_the_route_answered( @@ -313,7 +313,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s ( pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"), pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"), - pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"), + pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 503, "500", id="chat-anthropic"), ), ) def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status( @@ -361,7 +361,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes ( pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"), pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"), - pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"), + pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat-anthropic"), ), ) def test_the_otel_v2_server_span_carries_the_status_the_route_answered( diff --git a/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py b/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py index 6879d1f8acd..d4c0e2651d8 100644 --- a/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py +++ b/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py @@ -482,15 +482,20 @@ def test_duplicated_search_result_blocks_rebuild_the_call_once(gateway: Gateway, @pytest.mark.parametrize( - ("status", "kind", "caller_status"), + ("status", "kind", "caller_status", "caller_error_type", "caller_error_code"), ( - pytest.param(400, "invalid_request_error", 400, id="400"), - pytest.param(401, "authentication_error", 401, id="401"), - pytest.param(529, "overloaded_error", 500, id="529"), + pytest.param(400, "invalid_request_error", 400, None, None, id="400"), + pytest.param(401, "authentication_error", 401, None, None, id="401"), + pytest.param(529, "overloaded_error", 503, "service_unavailable_error", "server_is_overloaded", id="529"), ), ) def test_provider_error_on_the_replay_reaches_the_caller( - gateway: Gateway, status: int, kind: str, caller_status: int + gateway: Gateway, + status: int, + kind: str, + caller_status: int, + caller_error_type: str | None, + caller_error_code: str | None, ) -> None: marker: Final = uuid.uuid4().hex @@ -508,6 +513,11 @@ def test_provider_error_on_the_replay_reaches_the_caller( received: Final = wire.drain() assert second.status_code == caller_status, second.text assert f"scripted {kind} {marker}" in second.text, second.text + if caller_error_type is not None: + payload: Final = JSON_OBJECT.validate_json(second.content) + error: Final = JSON_OBJECT.validate_python(payload["error"]) + assert error["type"] == caller_error_type, second.text + assert error["code"] == caller_error_code, second.text assert len(received) >= 2, received assert wire_assistant(received[1]) == list(SEARCH_TURN), received[1].body assert gateway.client.get("/health/liveliness").status_code == 200 diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 67edcf7f3e4..2502bd90bd6 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -622,10 +622,15 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output @pytest.mark.parametrize("route", ["converse", "invoke"]) -def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gateway: Gateway, route: str) -> None: +@pytest.mark.parametrize( + ("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"] +) +def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default( + gateway: Gateway, route: str, backend: str, thinks_by_default: bool +) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, wire, route, _HAIKU) + model: Final = _deployment(scenario, wire, route, backend) response: Final = gateway.request( "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) ) @@ -634,6 +639,9 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa assert not message.get("tool_calls"), response.text _, body = _one_call(wire, marker_hex) invoke: Final = route == "invoke" + if thinks_by_default: + _assert_native_without_tools(body, invoke, _SCHEMA) + return assert _native_schema(body, invoke) is None, body assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body diff --git a/tests/integration/providers/test_nvidia_riva_wire.py b/tests/integration/providers/test_nvidia_riva_wire.py index ef36c449761..56d54a87600 100644 --- a/tests/integration/providers/test_nvidia_riva_wire.py +++ b/tests/integration/providers/test_nvidia_riva_wire.py @@ -135,3 +135,12 @@ def test_a_stalled_offline_recognize_answers_408_at_the_deployment_timeout(gatew assert timed_out.status_code == 408, timed_out.text calls: Final = eventually(server.drain, lambda observed: len(observed) > 0, seconds=30) assert all(call.rpc is Rpc.RECOGNIZE and call.cancelled for call in calls), calls + + +@pytest.mark.parametrize("riva_offline", (False, True), ids=("streaming", "offline")) +def test_the_deployment_api_key_reaches_riva_as_exactly_one_bearer_token(gateway: Gateway, riva_offline: bool) -> None: + with riva_server(_WORDS) as server, gateway.scenario() as scenario: + model: Final = _riva_model(scenario, server, api_key="riva-deployment-key", riva_offline=riva_offline) + served: Final = _transcribe(gateway, scenario.key(), model) + assert served.status_code == 200, served.text + assert [call.authorization for call in server.drain()] == [("Bearer riva-deployment-key",)] diff --git a/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py b/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py index 4b25275285e..2e1bd73a4b4 100644 --- a/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py +++ b/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py @@ -809,7 +809,10 @@ def test_provider_error_on_a_marked_request_reaches_the_caller(gateway: Gateway, model: Final = _anthropic_deployment(scenario, wire) answer: Final = _send(gateway, _marked_system(model, marker)) labels: Final = anthropic_labels(_only_request(wire)) - assert answer.status == 500, answer.text + assert answer.status == 503, answer.text + error: Final = object_value(_JSON_OBJECT.validate_json(answer.text)["error"]) + assert error["type"] == "service_unavailable_error", answer.text + assert error["code"] == "server_is_overloaded", answer.text assert "overloaded_error" in answer.text, answer.text assert labels == [SYSTEM_LABEL] spend.landed(model, answer.call_id, None, status="failure") diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index db81ff70263..19928ecd6e0 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -34,14 +34,20 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state _JSON_TOOL: Final = "json_tool_call" +_UNFORCED_JSON_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) _USER_TOOL: Final = "lookup" _STRUCTURED_OUTPUTS_BETA: Final = "structured-outputs-2025-11-13" _TOOL_MODEL: Final = "claude-sonnet-5" _EFFORT_GATED_MODEL: Final = "claude-haiku-4-5" _NATIVE_MODEL: Final = "claude-opus-5-5" -_FORCED_TOOL_NATIVE_MODEL: Final = "claude-haiku-5-5" +_FORCED_TOOL_NATIVE_MODEL: Final = "claude-opus-4-8" +_THINKING_BY_DEFAULT_NATIVE_MODEL: Final = "claude-haiku-5-5" +_THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8" _UNMAPPED_MODEL: Final = "claude-scripted-9" _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5" +_BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8" _BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8" _THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" _PROJECT: Final = "scripted-project" @@ -608,11 +614,15 @@ def _one_delivery(wire: Wire, marker_hex: str) -> _Delivery: return sent -def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]: +def _json_tool(delivery: _Delivery) -> dict[str, JsonValue]: tools: Final = tuple(object_value(tool) for tool in list_value(delivery.body["tools"])) (json_tool,) = tuple(tool for tool in tools if tool["name"] == _JSON_TOOL) - assert json_tool.keys() == {"name", "input_schema"}, json_tool - return object_value(json_tool["input_schema"]) + assert json_tool.keys() - {"description"} == {"name", "input_schema"}, json_tool + return json_tool + + +def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]: + return object_value(_json_tool(delivery)["input_schema"]) def _folded_format(delivery: _Delivery) -> dict[str, JsonValue]: @@ -830,10 +840,10 @@ def _family_deployment(gateway: Gateway, scenario: Scenario, api_base: str, memb case "azure_ai_unmapped": return scenario.model(model=f"azure_ai/{_UNMAPPED_MODEL}", api_base=api_base, api_key=_AZURE_KEY) case "bedrock_invoke": - return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_INVOKE_MODEL) + return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_TOOL_PATH_MODEL) case "bedrock_mantle": return scenario.model( - model=f"bedrock_mantle/{_BEDROCK_INVOKE_MODEL}", + model=f"bedrock_mantle/{_BEDROCK_TOOL_PATH_MODEL}", api_base=api_base, api_key=_MANTLE_KEY, aws_region_name="us-east-1", @@ -1448,21 +1458,24 @@ def test_tool_path_stream_cut_before_the_tool_block_closes_still_delivers_the_an _one_delivery(wire, marker_hex) -@pytest.mark.parametrize("accepts_forced_tools", [True, False], ids=["accepts_forced_tools", "rejects_forced_tools"]) -def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_accepts_forced_tools( - gateway: Gateway, accepts_forced_tools: bool +@pytest.mark.parametrize( + ("tool_model", "forced"), + [(_THINKING_OFF_TOOL_MODEL, True), (_TOOL_MODEL, False), (None, False)], + ids=["accepts_forced_tools", "thinks_by_default", "rejects_forced_tools"], +) +def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_will_not_think( + gateway: Gateway, tool_model: str | None, forced: bool ) -> None: marker, marker_hex = _marker("plain") - backend: Final = ( - _TOOL_MODEL if accepts_forced_tools else _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude") - ) + backend: Final = tool_model or _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude") with wire_server(_peer) as wire, gateway.scenario() as scenario: model: Final = _vertex_deployment(gateway, scenario, wire.url, backend) response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker)) assert _chat_content(response) == _payload(marker_hex), response.text sent: Final = _one_delivery(wire, marker_hex) assert _json_tool_schema(sent)["properties"] == _SCHEMA["properties"], sent.body - assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if accepts_forced_tools else None), sent.body + assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if forced else None), sent.body + assert _json_tool(sent).get("description") == (None if forced else _UNFORCED_JSON_TOOL_DESCRIPTION), sent.body assert "output_config" not in sent.body and "output_format" not in sent.body, sent.body @@ -1566,6 +1579,25 @@ def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_ke sent: Final = _one_delivery(wire, marker_hex) assert not _structured(sent.body), sent.body assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + assert sent.body.get("tool_choice") == _FORCED_JSON_TOOL, sent.body + + +@pytest.mark.parametrize("provider", ["anthropic", "azure_ai"]) +def test_open_object_schema_takes_the_unforced_json_tool_on_a_model_that_thinks_by_default( + gateway: Gateway, provider: str +) -> None: + marker, marker_hex = _marker("values") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _native_deployment(gateway, scenario, wire.url, provider, _THINKING_BY_DEFAULT_NATIVE_MODEL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT) + ) + assert _chat_content(response) == _tool_input("values", marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert not _structured(sent.body), sent.body + assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + assert _json_tool(sent).get("description") == _UNFORCED_JSON_TOOL_DESCRIPTION, sent.body + assert "tool_choice" not in sent.body, sent.body @pytest.mark.parametrize( diff --git a/tests/integration/routing/test_least_busy_concurrent_decisions.py b/tests/integration/routing/test_least_busy_concurrent_decisions.py new file mode 100644 index 00000000000..bb923534db4 --- /dev/null +++ b/tests/integration/routing/test_least_busy_concurrent_decisions.py @@ -0,0 +1,90 @@ +import json +import threading +from collections import Counter +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +pytestmark = pytest.mark.timeout(240) + +_GROUP: Final = "least-busy-decider" +_DEPLOYMENTS: Final = ("least-busy-decider-a", "least-busy-decider-b") +_BURST: Final = 4 +_REPLY: Final = json.dumps( + { + "model": "Qwen/Qwen3-0.6B", + "answers": {"billing": {"type": "noul", "noul": 0.91}}, + "usage": {"input_tokens": 367, "output_tokens": 3}, + } +).encode() +_DECISION: Final = { + "model": _GROUP, + "input": "I was charged twice for order 4417.", + "questions": [{"type": "predicate", "name": "billing", "instructions": "Is this a billing issue?"}], +} + + +def _held_until_the_burst_arrives(arrived: threading.Barrier) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + arrived.wait() + return Reply(body=_REPLY) + + return respond + + +def _write_config(path: Path, provider_url: str) -> Path: + path.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": _GROUP, + "litellm_params": {"model": "hosted_vllm/Qwen/Qwen3-0.6B", "api_base": provider_url}, + "model_info": {"id": deployment_id}, + } + for deployment_id in _DEPLOYMENTS + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_model_info_refresh": True, + }, + "router_settings": {"routing_strategy": "least-busy", "num_retries": 0, "disable_cooldowns": True}, + } + ) + ) + return path + + +def _decide(proxy: Gateway) -> httpx.Response: + return proxy.client.post( + "/v1/decisions", json=_DECISION, headers={"Authorization": f"Bearer {proxy.key}"}, timeout=120 + ) + + +def test_least_busy_spreads_a_burst_of_decisions_calls_across_the_group(tmp_path: Path) -> None: + arrived: Final = threading.Barrier(_BURST, timeout=60) + with ( + wire_server(_held_until_the_burst_arrives(arrived)) as upstream, + gateway_from_environment() as rig, + owned_proxy( + rig, + tmp_path, + {"STORE_MODEL_IN_DB": "False"}, + config=_write_config(tmp_path / "least-busy.yaml", upstream.url), + ) as proxy, + ThreadPoolExecutor(max_workers=_BURST) as pool, + ): + responses: Final = tuple(pool.map(lambda _: _decide(proxy), range(_BURST))) + assert [response.status_code for response in responses] == [200] * _BURST, [r.text for r in responses] + assert Counter(response.headers["x-litellm-model-id"] for response in responses) == Counter( + {deployment_id: _BURST // len(_DEPLOYMENTS) for deployment_id in _DEPLOYMENTS} + ) diff --git a/tests/integration/routing/test_prompt_cache_pin_ttl.py b/tests/integration/routing/test_prompt_cache_pin_ttl.py index f6ee96a9206..733f7859ff0 100644 --- a/tests/integration/routing/test_prompt_cache_pin_ttl.py +++ b/tests/integration/routing/test_prompt_cache_pin_ttl.py @@ -9,7 +9,7 @@ from typing import Final import anthropic import openai import pytest -from integration._support.client import eventually, gateway_from_environment, held +from integration._support.client import eventually, gateway_from_environment, held, object_value from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.redis_process import owned_redis from integration._support.wire import Wire, wire_server @@ -56,11 +56,12 @@ from integration.routing._prompt_cache_pin_support import ( vertex_peer, wire_carries_one_hour, ) -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter from redis import Redis pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 300), pytest.mark.shares_worker_state] +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _CLAUDE: Final = "pin-claude" _SLOW: Final = "pin-slow" _MIXED: Final = "pin-mixed" @@ -539,7 +540,10 @@ def test_failed_first_turn_reports_the_error_and_writes_no_pin(rig: _Rig) -> Non "/v1/chat/completions", chat_body(_CLAUDE, _first_turn(new_marker(), f"{new_marker()} {OVERLOAD_TOKEN}", ONE_HOUR_MARK)), ) - assert response.status_code == 500, response.text + assert response.status_code == 503, response.text + error: Final = object_value(_JSON_OBJECT.validate_json(response.text)["error"]) + assert error["type"] == "service_unavailable_error", response.text + assert error["code"] == "server_is_overloaded", response.text assert "overloaded_error" in response.text, response.text healthy, _ = post_chat(rig.owned.gateway, chat_body(_CLAUDE, _first_turn(new_marker(), new_marker(), None))) spend_row_lands(healthy) diff --git a/tests/integration/routing/test_reasoning_effort_levels_wire.py b/tests/integration/routing/test_reasoning_effort_levels_wire.py index d15867ac572..efca26d38d0 100644 --- a/tests/integration/routing/test_reasoning_effort_levels_wire.py +++ b/tests/integration/routing/test_reasoning_effort_levels_wire.py @@ -1977,6 +1977,9 @@ _INVOKE_NATIVE_OUTPUT: Final[dict[str, JsonValue]] = { "format": {"type": "json_schema", "schema": _NATIVE_OUTPUT_SCHEMA}, } _JSON_TOOL: Final = "json_tool_call" +_UNFORCED_JSON_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) _CONVERSE_TOOL_SCHEMA: Final[dict[str, JsonValue]] = { key: value for key, value in _NATIVE_OUTPUT_SCHEMA.items() if key != "additionalProperties" } @@ -1994,7 +1997,7 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields( { "toolSpec": { "name": _JSON_TOOL, - "description": _JSON_TOOL, + "description": _UNFORCED_JSON_TOOL_DESCRIPTION, "inputSchema": {"json": _CONVERSE_TOOL_SCHEMA}, } } @@ -2004,7 +2007,15 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields( ) _INVOKE_WITH_JSON_TOOL: Final = _with_fields( _without(_invoke_chat("high", _ADAPTIVE, ()), "anthropic_beta"), - {"tools": [{"name": _JSON_TOOL, "input_schema": _NATIVE_OUTPUT_SCHEMA}]}, + { + "tools": [ + { + "name": _JSON_TOOL, + "description": _UNFORCED_JSON_TOOL_DESCRIPTION, + "input_schema": _NATIVE_OUTPUT_SCHEMA, + } + ] + }, ) diff --git a/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py b/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py index 54a6df02edf..294791128ab 100644 --- a/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py +++ b/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py @@ -27,7 +27,7 @@ _RESPONSE_FORMAT: Final[dict[str, JsonValue]] = { "json_schema": {"name": "risk", "schema": _SCHEMA}, } _ANTHROPIC_MODEL: Final = "anthropic/claude-scripted-9" -_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-sonnet-5" +_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-opus-4-8" def _payload(marker_hex: str) -> dict[str, JsonValue]: diff --git a/tests/integration/security/test_skill_scanner.py b/tests/integration/security/test_skill_scanner.py new file mode 100644 index 00000000000..e18378463ee --- /dev/null +++ b/tests/integration/security/test_skill_scanner.py @@ -0,0 +1,58 @@ +import asyncio +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import psutil +import pytest +from fastapi import HTTPException + +from cookbook.skill_scanner import server +from tests.integration._support.process import _wait_until_gone, group_members, stop_processes + + +@pytest.mark.parametrize("mode", ("deadline", "cap", "complete")) +async def test_adapter_bounds_worker_lifetime_and_reaps_its_process_group( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mode: str, +) -> None: + script: Final = ( + "import subprocess, sys, time\n" + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], " + "stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n" + "print(child.pid, flush=True)\n" + + ("pass\n" if mode == "complete" else "time.sleep(30)\n") + ) + child: Final = await asyncio.create_subprocess_exec( + sys.executable, "-c", script, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, start_new_session=True, + ) + try: + assert child.stdout is not None + descendant: Final = psutil.Process(int(await asyncio.wait_for(child.stdout.readline(), 5))) + if mode == "complete": + await asyncio.wait_for(child.wait(), 5) + output: Final = tmp_path / "report.json" + output.write_text('{"complete": true}') + + async def launch(*args: object, **kwargs: object) -> asyncio.subprocess.Process: + return child + + monkeypatch.setattr(server.asyncio, "create_subprocess_exec", launch) + monkeypatch.setattr(server, "SCAN_TIMEOUT", 0.05 if mode == "cap" else 30) + deadline: Final = ( + None if mode == "cap" + else datetime.now(timezone.utc) + timedelta(seconds=0.05 if mode == "deadline" else 30) + ) + result: Final = server.run_worker("unused.zip", output, False, deadline) + if mode == "complete": + assert await asyncio.wait_for(result, 5) == {"complete": True} + else: + with pytest.raises(HTTPException) as raised: + await asyncio.wait_for(result, 5) + assert raised.value.status_code == 504 + assert child.returncode is not None + _wait_until_gone(descendant, 3) + finally: + await asyncio.to_thread(stop_processes, group_members(child.pid), 0.1) + await child.wait() diff --git a/tests/integration/spend/golden/daily_activity_user_aggregated.json b/tests/integration/spend/golden/daily_activity_user_aggregated.json index 185d333965a..3189c000035 100644 --- a/tests/integration/spend/golden/daily_activity_user_aggregated.json +++ b/tests/integration/spend/golden/daily_activity_user_aggregated.json @@ -1,5 +1,5 @@ { - "metadata": {"entity_total_api_keys":null, + "metadata": {"entity_total_api_keys":{"user-1": 5}, "api_key_limit": 100, "has_more": false, "page": 1, @@ -346,7 +346,181 @@ } } }, - "entities": {}, + "entities": { + "user-1": { + "api_key_breakdown": { + "key-a": { + "metadata": { + "key_alias": "alias-a", + "key_exists": true, + "team_id": "team-1", + "user_email": "user@example.com", + "user_id": "user-1" + }, + "metrics": { + "api_requests": 1, + "autorouter_savings_spend": 0.0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1, + "completion_tokens": 2, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "failed_requests": 0, + "flat_cost": 0.0, + "ptu_hours": 0.0, + "gateway_injected_caching_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, + "prompt_tokens": 2, + "spend": 100.0, + "successful_requests": 1, + "timed_requests": 0, + "total_response_time_ms": 0, + "total_tokens": 4 + } + }, + "key-b": { + "metadata": { + "key_alias": "alias-b", + "key_exists": true, + "team_id": "team-1", + "user_email": "user@example.com", + "user_id": "user-1" + }, + "metrics": { + "api_requests": 1, + "autorouter_savings_spend": 0.0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1, + "completion_tokens": 2, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "failed_requests": 0, + "flat_cost": 0.0, + "ptu_hours": 0.0, + "gateway_injected_caching_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, + "prompt_tokens": 3, + "spend": 90.0, + "successful_requests": 1, + "timed_requests": 0, + "total_response_time_ms": 0, + "total_tokens": 5 + } + }, + "key-c": { + "metadata": { + "key_alias": "alias-c", + "key_exists": true, + "team_id": "team-1", + "user_email": "user@example.com", + "user_id": "user-1" + }, + "metrics": { + "api_requests": 1, + "autorouter_savings_spend": 0.0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1, + "completion_tokens": 2, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "failed_requests": 0, + "flat_cost": 0.0, + "ptu_hours": 0.0, + "gateway_injected_caching_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, + "prompt_tokens": 4, + "spend": 80.0, + "successful_requests": 1, + "timed_requests": 0, + "total_response_time_ms": 0, + "total_tokens": 6 + } + }, + "key-cache": { + "metadata": { + "key_alias": "alias-cache", + "key_exists": true, + "team_id": "team-1", + "user_email": "user@example.com", + "user_id": "user-1" + }, + "metrics": { + "api_requests": 1, + "autorouter_savings_spend": 0.0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1, + "completion_tokens": 2, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "failed_requests": 0, + "flat_cost": 0.0, + "ptu_hours": 0.0, + "gateway_injected_caching_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, + "prompt_tokens": 1000, + "spend": 2.0, + "successful_requests": 1, + "timed_requests": 0, + "total_response_time_ms": 0, + "total_tokens": 1002 + } + }, + "key-target": { + "metadata": { + "key_alias": "deleted-target", + "key_exists": false, + "team_id": "team-1", + "user_email": "user@example.com", + "user_id": "user-1" + }, + "metrics": { + "api_requests": 1, + "autorouter_savings_spend": 0.0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 2, + "completion_tokens": 2, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "failed_requests": 0, + "flat_cost": 0.0, + "ptu_hours": 0.0, + "gateway_injected_caching_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, + "prompt_tokens": 5, + "spend": 1.0, + "successful_requests": 1, + "timed_requests": 0, + "total_response_time_ms": 0, + "total_tokens": 7 + } + } + }, + "metadata": { + "user_alias": null, + "user_email": "user@example.com" + }, + "metrics": { + "api_requests": 5, + "autorouter_savings_spend": 0.0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 6, + "completion_tokens": 10, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "failed_requests": 0, + "flat_cost": 0.0, + "ptu_hours": 0.0, + "gateway_injected_caching_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, + "prompt_tokens": 1014, + "spend": 1273.0, + "successful_requests": 5, + "timed_requests": 0, + "total_response_time_ms": 0, + "total_tokens": 1024 + } + } + }, "mcp_servers": {}, "model_groups": { "model-cache": { diff --git a/tests/integration/translation/chat_completions/bases/bedrock_converse.py b/tests/integration/translation/chat_completions/bases/bedrock_converse.py index 6540bd09d64..777e0d04db5 100644 --- a/tests/integration/translation/chat_completions/bases/bedrock_converse.py +++ b/tests/integration/translation/chat_completions/bases/bedrock_converse.py @@ -1407,6 +1407,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], "inferenceConfig": {"maxTokens": 64}, "system": [{"text": "You are a terse assistant."}], diff --git a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py index 07e4358ff97..5cb7a7eee8e 100644 --- a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py +++ b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py @@ -6,7 +6,7 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.chat_completions.bases.anthropic import CLAUDE_HAIKU_5_5_TEST_CASE +from integration.translation.chat_completions.bases.anthropic import CLAUDE_OPUS_4_8_TEST_CASE from integration.translation.runner import assert_translation PLACE_SCHEMA: Final = { @@ -24,25 +24,25 @@ PLACE_TOOL_USE: Final = { "caller": {"type": "direct"}, } -CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( - CLAUDE_HAIKU_5_5_TEST_CASE, +CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE: Final = replace( + CLAUDE_OPUS_4_8_TEST_CASE, scenario="json_schema", litellm_request={ - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "max_tokens": 128, "messages": [{"role": "user", "content": PLACE_PROMPT}], "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, "cache": {"no-cache": True}, }, expected_provider_request={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], "max_tokens": 128, "tool_choice": {"name": "json_tool_call", "type": "tool"}, "tools": [{"name": "json_tool_call", "input_schema": PLACE_SCHEMA}], }, mock_provider_response={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "id": "msg_011CgCoP2UR3vcM5ZWmEBd8D", "type": "message", "role": "assistant", @@ -68,7 +68,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( "id": ANY, "created": ANY, "object": "chat.completion", - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [ { "finish_reason": "stop", @@ -97,11 +97,11 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( }, ) -CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( - CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, +CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( + CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, scenario="json_schema_tool_named_envelope", mock_provider_response={ - **CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE.mock_provider_response, + **CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE.mock_provider_response, "content": [ { **PLACE_TOOL_USE, @@ -114,7 +114,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( @pytest.mark.parametrize( "case", - [CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], + [CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], ids=lambda case: case.id, ) def test_chat_completions_response_format_anthropic( diff --git a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py index fca8de69f00..a41752a96cf 100644 --- a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py +++ b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py @@ -6,7 +6,7 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import SSEFrame, StreamingTranslationTestCase from integration.translation.chat_completions.bases.anthropic import ( - CLAUDE_HAIKU_5_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, ) from integration.translation.runner import assert_streaming_translation @@ -158,27 +158,27 @@ def _place_chunk(delta: dict[str, JsonValue]) -> SSEFrame: "id": ANY, "object": "chat.completion.chunk", "created": ANY, - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [{"index": 0, "delta": delta}], } ) -CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( +CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( scenario="json_schema_cut_stream", - litellm_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.litellm_endpoint, + litellm_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.litellm_endpoint, litellm_request={ - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "max_tokens": 128, "messages": [{"role": "user", "content": PLACE_PROMPT}], "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, "cache": {"no-cache": True}, "stream": True, }, - expected_provider_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_endpoint, - expected_provider_headers=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_headers, + expected_provider_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_endpoint, + expected_provider_headers=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_headers, expected_provider_request={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], "max_tokens": 128, "tool_choice": {"name": "json_tool_call", "type": "tool"}, @@ -191,7 +191,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT data={ "type": "message_start", "message": { - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "id": "msg_011CgCoP5RkexwE6N7PRZEca", "type": "message", "role": "assistant", @@ -263,7 +263,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT "id": ANY, "object": "chat.completion", "created": ANY, - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [ { "index": 0, @@ -276,7 +276,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT @pytest.mark.parametrize( "case", - [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE], + [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE], ids=lambda case: case.id, ) def test_chat_completions_streaming_anthropic( diff --git a/tests/integration/translation/messages/bases/bedrock_converse.py b/tests/integration/translation/messages/bases/bedrock_converse.py index 6a115160fbc..3d2b731d7c0 100644 --- a/tests/integration/translation/messages/bases/bedrock_converse.py +++ b/tests/integration/translation/messages/bases/bedrock_converse.py @@ -113,6 +113,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], "inferenceConfig": {"maxTokens": 64}, "system": [{"text": "You are a terse assistant."}], diff --git a/tests/integration/translation/responses/bases/bedrock_converse.py b/tests/integration/translation/responses/bases/bedrock_converse.py index b0a2c881878..d15f6325daf 100644 --- a/tests/integration/translation/responses/bases/bedrock_converse.py +++ b/tests/integration/translation/responses/bases/bedrock_converse.py @@ -190,6 +190,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], "inferenceConfig": {"maxTokens": 64}, "system": [{"text": "You are a terse assistant."}], diff --git a/tests/unit/accounting/test_delivery_projection.py b/tests/unit/accounting/test_delivery_projection.py new file mode 100644 index 00000000000..0b1f36d6b48 --- /dev/null +++ b/tests/unit/accounting/test_delivery_projection.py @@ -0,0 +1,85 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone +from typing import Final +from urllib.parse import unquote + +from litellm.accounting.delivery_projection import entity_transactions, project_deliveries +from litellm.accounting.projection import Contribution +from litellm.repositories.accounting_delivery_repository import DeliveryProjection + +_NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) + + +def _payload(spend: float, started: datetime = _NOW, window_start: datetime = _NOW) -> Mapping[str, object]: + return { + "request_id": f"request-{spend}-{started.isoformat()}", + "startTime": started, + "api_key": "key", + "user": "user/1", + "team_id": "team", + "organization_id": "org::1", + "spend": spend, + "metadata": json.dumps( + { + "accounting_entities": { + "api_key": "key", + "user": "user/1", + "team_id": "team", + "organization_id": "org::1", + }, + "accounting_windows": [ + { + "entity_type": "key", + "entity_id": "key", + "window_duration": "1d", + "window_start": window_start.isoformat(), + } + ], + } + ), + } + + +def _spend(contributions: tuple[Contribution, ...], table: str) -> tuple[float, ...]: + return tuple(float(dict(item.increments)["spend"]) for item in contributions if item.table == table) + + +def test_deliveries_for_one_destination_sum_and_keep_the_earliest_request_start() -> None: + later: Final = _NOW + timedelta(hours=1) + + contributions: Final = project_deliveries( + (DeliveryProjection(_payload(0.5, started=later)), DeliveryProjection(_payload(0.25))) + ) + + assert _spend(contributions, "LiteLLM_VerificationToken") == (0.75,) + windows: Final = tuple(item for item in contributions if item.table == "LiteLLM_BudgetWindowSpend") + assert len(windows) == 1 + assert dict(windows[0].increments) == {"spend": 0.75} + assert dict(windows[0].values)["started_at"] == str(_NOW) + + +def test_the_same_window_in_a_new_period_is_charged_to_its_own_row() -> None: + next_day: Final = _NOW + timedelta(days=1) + + contributions: Final = project_deliveries( + (DeliveryProjection(_payload(0.25)), DeliveryProjection(_payload(0.5, window_start=next_day))) + ) + + windows: Final = sorted( + (dict(item.values)["window_start"], dict(item.increments)["spend"]) + for item in contributions + if item.table == "LiteLLM_BudgetWindowSpend" + ) + assert windows == [(_NOW.isoformat(), 0.25), (next_day.isoformat(), 0.5)] + + +def test_entity_transactions_key_memberships_the_way_the_writer_applies_them() -> None: + transactions: Final = entity_transactions(project_deliveries((DeliveryProjection(_payload(0.25)),))) + + assert transactions["key_list_transactions"] == {"key": 0.25} + assert transactions["org_list_transactions"] == {"org::1": 0.25} + assert transactions["team_member_list_transactions"] == {"team_id::team::user_id::user/1": 0.25} + ((org_member_key, org_member_spend),) = transactions["org_member_list_transactions"].items() + _, quoted_org_id, _, quoted_user_id = org_member_key.split("::") + assert (unquote(quoted_org_id), unquote(quoted_user_id), org_member_spend) == ("org::1", "user/1", 0.25) diff --git a/tests/unit/accounting/test_measurements.py b/tests/unit/accounting/test_measurements.py deleted file mode 100644 index b65c3076c5d..00000000000 --- a/tests/unit/accounting/test_measurements.py +++ /dev/null @@ -1,235 +0,0 @@ -from datetime import datetime, timezone -from decimal import Decimal -from typing import Final - -import pytest - -from litellm.accounting import ( - CostEvent, - Dimension, - DimensionName, - InboundUsageEvent, - Measurement, - Metric, - MetricName, - Outcome, - OutboundUsageEvent, - PricingSnapshot, - measurements_for, -) -from litellm.types.utils import CallTypes - -_START: Final = datetime(2026, 1, 1, 23, 59, 59, tzinfo=timezone.utc) -_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc) -_DIMENSIONS: Final = ( - Dimension(name=DimensionName.TEAM, value="team"), - Dimension(name=DimensionName.TAG, value="alpha"), - Dimension(name=DimensionName.TAG, value="beta"), -) - - -def _snapshot(amount: Decimal) -> PricingSnapshot: - return PricingSnapshot( - calculator_version="test", - model="test-model", - provider=None, - pricing_source="upstream_header", - priced_model_key="test-model", - upstream_reported_cost=amount, - total_cost=amount, - ) - - -@pytest.mark.parametrize( - ("outcome", "outcome_metric"), - [ - (Outcome.SUCCEEDED, MetricName.SUCCESSFUL_REQUESTS), - (Outcome.FAILED, MetricName.FAILED_REQUESTS), - (Outcome.CANCELLED, MetricName.FAILED_REQUESTS), - ], -) -def test_caller_measurements_keep_both_request_and_outcome_counts(outcome, outcome_metric): - event = InboundUsageEvent( - event_id="caller-event", - request_id="caller", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=outcome, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)), Metric(name=outcome_metric, value=Decimal(1))), - ) - assert measurements_for(event) == ( - Measurement( - source_event_id="caller-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.REQUESTS, value=Decimal(1)), - ), - Measurement( - source_event_id="caller-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=outcome_metric, value=Decimal(1)), - ), - ) - - -def test_usage_measurements_keep_every_metric_in_the_start_time_bucket(): - event = OutboundUsageEvent( - event_id="usage-event", - attempt_id="attempt", - request_id="caller", - retry_of_attempt_id=None, - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=_DIMENSIONS, - metrics=( - Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)), - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)), - Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)), - Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)), - ), - ) - assert measurements_for(event) == ( - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)), - ), - ) - - -@pytest.mark.parametrize("metrics", [None, ()]) -def test_unknown_or_empty_usage_does_not_fabricate_zero_measurements(metrics): - event = OutboundUsageEvent( - event_id="usage-event", - attempt_id="attempt", - request_id=None, - retry_of_attempt_id=None, - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=(), - metrics=metrics, - ) - assert measurements_for(event) == () - - -@pytest.mark.parametrize("amount", [Decimal(0), Decimal("0.1234567890123456789")]) -def test_cost_measurements_preserve_authoritative_amount_identity_and_occurrence(amount): - event = CostEvent( - event_id="cost-event", - usage_event_id="usage-event", - occurred_at=_END, - dimensions=_DIMENSIONS, - charges=(Metric(name=MetricName.SPEND, value=amount, unit="USD"),), - snapshot=_snapshot(amount), - ) - assert measurements_for(event) == ( - Measurement( - source_event_id="cost-event", - occurred_at=_END, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.SPEND, value=amount, unit="USD"), - ), - ) - - -def test_billed_retry_and_success_contribute_one_caller_request_and_both_charges(): - caller = InboundUsageEvent( - event_id="caller-event", - request_id="caller", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), - ) - retry = OutboundUsageEvent( - event_id="retry-usage", - attempt_id="retry", - request_id="caller", - retry_of_attempt_id=None, - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.FAILED, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(4)),), - ) - success = OutboundUsageEvent( - event_id="success-usage", - attempt_id="success", - request_id="caller", - retry_of_attempt_id="retry", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),), - ) - retry_cost = CostEvent( - event_id="retry-cost", - usage_event_id="retry-usage", - occurred_at=_START, - dimensions=_DIMENSIONS, - charges=(Metric(name=MetricName.SPEND, value=Decimal("0.025"), unit="USD"),), - snapshot=_snapshot(Decimal("0.025")), - ) - success_cost = CostEvent( - event_id="success-cost", - usage_event_id="success-usage", - occurred_at=_START, - dimensions=_DIMENSIONS, - charges=(Metric(name=MetricName.SPEND, value=Decimal("0.125"), unit="USD"),), - snapshot=_snapshot(Decimal("0.125")), - ) - measurements = ( - *measurements_for(caller), - *measurements_for(retry), - *measurements_for(success), - *measurements_for(retry_cost), - *measurements_for(success_cost), - ) - assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.REQUESTS) == Decimal(1) - assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.TOTAL_TOKENS) == Decimal(16) - assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.SPEND) == Decimal("0.150") - assert {m.source_event_id for m in measurements} == { - "caller-event", - "retry-usage", - "success-usage", - "retry-cost", - "success-cost", - } diff --git a/tests/unit/accounting/test_metrics.py b/tests/unit/accounting/test_metrics.py new file mode 100644 index 00000000000..49b46eb36c2 --- /dev/null +++ b/tests/unit/accounting/test_metrics.py @@ -0,0 +1,28 @@ +from prometheus_client.metrics import MetricWrapperBase + +from litellm.accounting import metrics +from litellm.accounting.metrics import record_deliveries +from litellm.repositories.accounting_delivery_repository import DeliveryMetrics + + +def _sample(metric: MetricWrapperBase, suffix: str = "") -> float: + family = metric.collect()[0] + return next(sample.value for sample in family.samples if sample.name == family.name + suffix) + + +def test_delivery_metrics_publish_backlog_and_clamp_clock_skew() -> None: + batches_before = _sample(metrics._BATCH_TIME, "_count") + + record_deliveries( + DeliveryMetrics(pending_count=4, parked_count=1, oldest_pending_seconds=-3.0, batch_count=7, table_bytes=8192), + 0.5, + ) + + assert ( + _sample(metrics._PENDING), + _sample(metrics._PARKED), + _sample(metrics._OLDEST), + _sample(metrics._BATCH_COUNT), + _sample(metrics._TABLE_BYTES), + ) == (4.0, 1.0, 0.0, 7.0, 8192.0) + assert _sample(metrics._BATCH_TIME, "_count") == batches_before + 1 diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py index ec0ab44e505..17186a89922 100644 --- a/tests/unit/accounting/test_projection.py +++ b/tests/unit/accounting/test_projection.py @@ -429,3 +429,129 @@ def test_guardrail_usage_is_dated_in_utc() -> None: assert [dict(item.key)["date"] for item in projected if item.table == "LiteLLM_DailyGuardrailUsageUnits"] == [ "2026-11-03" ] + + +@pytest.mark.parametrize( + ("untrusted", "metadata"), + [ + ({"request_tags": json.dumps(["x" * 16384, "normal"])}, {}), + ({"end_user": "x" * 3072}, {}), + ({}, {"compression_savings": {"tokens_saved": 2**63}}), + ], +) +def test_untrusted_analytics_values_never_block_the_charge( + untrusted: Mapping[str, str], metadata: Mapping[str, object] +) -> None: + row: Final[Mapping[str, object]] = { + "request_id": "untrusted", + "startTime": _NOW, + "api_key": "key", + "user": "user", + "team_id": "team", + "model": "unpriced-projection-test", + "model_group": "group", + "custom_llm_provider": "openai", + "spend": 0.5, + **untrusted, + "metadata": { + **metadata, + "accounting_entities": { + "api_key": "key", + "user": "user", + "team_id": "team", + "end_user": untrusted.get("end_user"), + }, + }, + } + contributions: Final = project(row) + texts: Final = tuple(value for item in contributions for _, value in (*item.key, *item.values) if value) + numbers: Final = tuple(value for item in contributions for _, value in item.increments) + assert {"LiteLLM_VerificationToken", "LiteLLM_TeamTable", "LiteLLM_UserTable"} <= { + item.table for item in contributions + } + assert "LiteLLM_DailyUserSpend" in {item.table for item in contributions} + assert all(len(text.encode()) <= 2000 for text in texts) + assert all(abs(number) <= 2**40 for number in numbers) + + +def test_stored_payload_keeps_guardrail_billing_but_not_guardrail_content() -> None: + payload: Final = get_logging_payload( + kwargs={ + "model": "projection-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "guardrail-request", + "litellm_params": {"metadata": {"user_api_key": "key", "user_api_key_user_id": "user"}}, + }, + response_obj=ModelResponse( + id="guardrail-request", model="projection-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ) + guardrail: Final = { + "guardrail_name": "pii", + "guardrail_usage": {"requests": 1}, + "guardrail_cost_by_unit": {"requests": 0.25}, + "guardrail_response": "card 4111 1111 1111 1111", + "match_details": [{"matched": "4111 1111 1111 1111"}], + } + headroom: Final = { + "guardrail_name": "compress", + "guardrail_provider": "headroom", + "guardrail_response": {"tokens_saved": 7, "compressed_prompt": "card 4111 1111 1111 1111"}, + } + metadata: Final = {**_FIELDS.validate_json(payload["metadata"]), "guardrail_information": [guardrail, headroom]} + accounting_input: Final[SpendLogsPayload] = { + **payload, + "spend": 0.5, + "metadata": json.dumps(metadata), + "request_tags": '["nul\x00tag"]', + } + accounting: Final = build_accounting_payload( + accounting_input, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None + ) + stored: Final = json.dumps(accounting, default=str) + restored: Final = _FIELDS.validate_json(stored) + usage_rows: Final = tuple(item for item in project(restored) if item.table == "LiteLLM_DailyGuardrailUsageUnits") + assert "4111" not in stored + assert "\\u0000" not in stored + assert [dict(item.increments) for item in usage_rows] == [{"units": 1, "cost": 0.25, "untracked_units": 0}] + daily_user: Final = next(item for item in project(restored) if item.table == "LiteLLM_DailyUserSpend") + assert dict(daily_user.increments)["compression_saved_tokens"] == 7 + + +def test_caller_supplied_savings_that_cannot_be_computed_never_drop_the_charge(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + + monkeypatch.setitem( + litellm.model_cost, + "savings-overflow-test", + {"input_cost_per_token": 0.002, "output_cost_per_token": 0.004, "litellm_provider": "openai", "mode": "chat"}, + ) + payload: Final[SpendLogsPayload] = { + **get_logging_payload( + kwargs={ + "model": "savings-overflow-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "overflow-request", + "litellm_params": {"metadata": {"user_api_key": "key"}}, + }, + response_obj=ModelResponse( + id="overflow-request", model="savings-overflow-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ), + "spend": 0.5, + "metadata": json.dumps({"compression_savings": {"tokens_saved": 10**400}}), + } + accounting: Final = build_accounting_payload( + payload, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None + ) + restored: Final = _FIELDS.validate_json(json.dumps(accounting, default=str)) + assert dict(restored["metadata"])["accounting_savings"]["compression"] == 0.0 + assert [dict(item.increments)["spend"] for item in project(restored) if item.table == "LiteLLM_VerificationToken"] == [ + 0.5 + ] diff --git a/tests/unit/accounting/test_types.py b/tests/unit/accounting/test_types.py deleted file mode 100644 index c1be39ab111..00000000000 --- a/tests/unit/accounting/test_types.py +++ /dev/null @@ -1,482 +0,0 @@ -import re -from collections.abc import Mapping -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from enum import Enum -from pathlib import Path -from typing import Final - -import pytest -from pydantic import ValidationError - -from litellm.accounting import ( - AdmissionDecision, - AggregateQuery, - AggregateRow, - CostEvent, - DeliveryState, - DeliveryTarget, - Dimension, - DimensionFilter, - DimensionName, - Granularity, - InboundUsageEvent, - Metric, - MetricName, - Multiplier, - Outcome, - OutboundUsageEvent, - PricedQuantity, - PricingSnapshot, - Rate, - validate_aggregate_rows, -) -from litellm.accounting.types import PricingUnit -from litellm.types.utils import CallTypes - -_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) -_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc) -_REQUEST: Final = Metric(name=MetricName.REQUESTS, value=Decimal(1)) -_TOKEN: Final = Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)) -_SPEND: Final = Metric(name=MetricName.SPEND, value=Decimal("0.1234567890123456789"), unit="USD") -_TEAM: Final = Dimension(name=DimensionName.TEAM, value="team") - - -def _inbound(metrics=(_REQUEST,), outcome=Outcome.SUCCEEDED): - return InboundUsageEvent( - event_id="inbound:request", - request_id="request", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_START, - outcome=outcome, - dimensions=(_TEAM,), - metrics=metrics, - ) - - -def _outbound(metrics=(_TOKEN,), **changes): - return OutboundUsageEvent.model_validate( - { - "event_id": "outbound:attempt", - "attempt_id": "attempt", - "request_id": "request", - "retry_of_attempt_id": None, - "call_type": CallTypes.acompletion, - "started_at": _START, - "ended_at": _END, - "outcome": Outcome.SUCCEEDED, - "dimensions": (_TEAM,), - "metrics": metrics, - **changes, - } - ) - - -def _query(**changes): - return AggregateQuery.model_validate( - { - "metric": MetricName.SPEND, - "filters": (DimensionFilter(name=DimensionName.TEAM, values=("one", "two")),), - "group_by": (DimensionName.MODEL,), - "start": _START, - "end": _END, - "granularity": Granularity.DAY, - **changes, - } - ) - - -def test_wire_values_preserve_decimal_precision_and_normalize_aware_times(): - offset = timezone(timedelta(hours=-8)) - event = CostEvent( - event_id="cost:attempt", - usage_event_id="outbound:attempt", - occurred_at=datetime(2025, 12, 31, 16, tzinfo=offset), - dimensions=(_TEAM,), - charges=(_SPEND,), - snapshot=_snapshot(), - ) - wire = event.model_dump(mode="json") - assert wire["occurred_at"] == "2026-01-01T00:00:00Z" - assert wire["charges"] == [{"name": "spend", "value": "0.1234567890123456789", "unit": "USD"}] - assert event.occurred_at == _START - assert CostEvent.model_validate_json(event.model_dump_json()).charges[0].value == _SPEND.value - - -@pytest.mark.parametrize( - ("name", "value", "unit", "message"), - [ - (MetricName.SPEND, Decimal(1), "count", "SPEND requires USD"), - (MetricName.REQUESTS, Decimal(1), "USD", "SPEND requires USD"), - (MetricName.PROMPT_TOKENS, Decimal("1.5"), "count", "integral"), - (MetricName.SPEND, Decimal("NaN"), "USD", "finite"), - (MetricName.SPEND, Decimal("Infinity"), "USD", "finite"), - (MetricName.SPEND, Decimal(-1), "USD", "greater than or equal"), - (MetricName.SPEND, 0.1, "USD", "instance of Decimal"), - ], -) -def test_metrics_reject_wrong_units_nonfinite_values_and_binary_float_inputs(name, value, unit, message): - with pytest.raises(ValidationError, match=message): - Metric(name=name, value=value, unit=unit) - - -@pytest.mark.parametrize("metrics", [(_TOKEN,), (_SPEND,), ()]) -def test_inbound_activity_cannot_own_usage_or_cost_or_omit_request_count(metrics): - with pytest.raises(ValidationError, match="Inbound events"): - _inbound(metrics) - - -@pytest.mark.parametrize( - ("outcome", "metric"), - [ - (Outcome.FAILED, Metric(name=MetricName.SUCCESSFUL_REQUESTS, value=Decimal(1))), - (Outcome.SUCCEEDED, Metric(name=MetricName.FAILED_REQUESTS, value=Decimal(1))), - ], -) -def test_inbound_outcome_counts_cannot_contradict_outcome(outcome, metric): - with pytest.raises(ValidationError, match="count must match outcome"): - _inbound((_REQUEST, metric), outcome) - - -@pytest.mark.parametrize("metrics", [(_REQUEST,), (_SPEND,)]) -def test_outbound_attempts_cannot_own_caller_counts_or_charges(metrics): - with pytest.raises(ValidationError, match="Outbound events own only"): - _outbound(metrics) - - -@pytest.mark.parametrize( - ("changes", "message"), - [ - ({"started_at": datetime(2026, 1, 1)}, "timezone"), - ({"ended_at": _START - timedelta(seconds=1)}, "End must follow start"), - ({"retry_of_attempt_id": "attempt"}, "cannot retry itself"), - ({"dimensions": (_TEAM, _TEAM)}, "Duplicate dimension values"), - ({"dimensions": (_TEAM, Dimension(name=DimensionName.TEAM, value="other"))}, "Only tag and model access group"), - ( - { - "metrics": ( - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(5)), - Metric(name=MetricName.REASONING_TOKENS, value=Decimal(6)), - ) - }, - "reasoning_tokens cannot exceed completion_tokens", - ), - ( - { - "metrics": ( - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(5)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(6)), - ) - }, - "cache_creation_input_tokens_above_1hr cannot exceed cache_creation_input_tokens", - ), - ({"event_id": " "}, "pattern"), - ({"metrics": (_TOKEN, _TOKEN)}, "Duplicate metric names"), - ], -) -def test_outbound_events_reject_invalid_identity_time_and_duplicate_attribution(changes, message): - with pytest.raises(ValidationError, match=message): - _outbound(**changes) - - -def test_unknown_and_known_zero_usage_are_distinct_and_values_are_immutable(): - unknown = _outbound(None) - zero = _outbound((Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(0)),)) - assert unknown.metrics is None - assert zero.metrics is not None and zero.metrics[0].value == 0 - assert _inbound().started_at == _inbound().ended_at - with pytest.raises(ValidationError, match="frozen"): - zero.attempt_id = "different" - with pytest.raises(ValidationError, match="frozen"): - zero.metrics[0].value = Decimal(10) - - -@pytest.mark.parametrize("charges", [(), (_TOKEN,), (_SPEND, _SPEND)]) -def test_cost_events_require_one_authoritative_spend(charges): - with pytest.raises(ValidationError, match=r"authoritative USD spend|Duplicate metric"): - CostEvent( - event_id="cost", - usage_event_id="usage", - occurred_at=_START, - dimensions=(), - charges=charges, - snapshot=_snapshot(), - ) - - -def test_cost_event_cannot_reuse_its_usage_identity(): - with pytest.raises(ValidationError, match="distinct identities"): - CostEvent( - event_id="usage", - usage_event_id="usage", - occurred_at=_START, - dimensions=(), - charges=(_SPEND,), - snapshot=_snapshot(), - ) - - -@pytest.mark.parametrize("granularity", [Granularity.DAY, None]) -def test_daily_and_whole_interval_queries_preserve_filter_and_group_semantics(granularity): - query = _query(granularity=granularity) - assert query.granularity is granularity - assert query.filters[0].values == ("one", "two") - assert query.group_by == (DimensionName.MODEL,) - assert query.start == _START and query.end == _END - - -@pytest.mark.parametrize( - ("changes", "message"), - [ - ({"end": _START}, "End must follow start"), - ({"start": _START + timedelta(hours=1)}, "UTC day boundaries"), - ({"end": _END + timedelta(microseconds=1)}, "UTC day boundaries"), - ({"granularity": "hour"}, "day"), - ({"group_by": (DimensionName.TEAM, DimensionName.TEAM)}, "Duplicate group-by"), - ({"filters": (DimensionFilter(name=DimensionName.TEAM, values=("one",)),) * 2}, "one IN filter"), - ({"filters": ({"name": "team", "values": ()},)}, "at least 1"), - ], -) -def test_queries_reject_unsupported_or_ambiguous_ranges_and_filters(changes, message): - with pytest.raises(ValidationError, match=message): - _query(**changes) - - -def test_rows_allow_adjacent_buckets_and_overlapping_distinct_groups(): - first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM,), value=Decimal(1)) - second = AggregateRow(start=_END, end=_END + timedelta(days=1), dimensions=(_TEAM,), value=Decimal(2)) - other = AggregateRow(start=_START, end=_END, dimensions=(), value=Decimal(3)) - rows = (second, other, first) - assert validate_aggregate_rows(rows) is rows - assert validate_aggregate_rows(()) == () - - -def test_rows_reject_overlap_within_a_group_regardless_of_dimension_order(): - model = Dimension(name=DimensionName.MODEL, value="model") - first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM, model), value=Decimal(1)) - overlapping = AggregateRow(start=_START, end=_END, dimensions=(model, _TEAM), value=Decimal(2)) - with pytest.raises(ValueError, match="overlap within a dimension group"): - validate_aggregate_rows((first, overlapping)) - - -def test_rows_reject_empty_intervals(): - with pytest.raises(ValidationError, match="End must follow start"): - AggregateRow(start=_START, end=_START, dimensions=(), value=Decimal(0)) - - -def test_admission_decision_preserves_a_denial_reason(): - decision = AdmissionDecision(allowed=False, reason="Team budget exhausted") - assert not decision.allowed - assert decision.reason == "Team budget exhausted" - - -def _snapshot(**changes: object) -> PricingSnapshot: - return PricingSnapshot.model_validate( - { - "calculator_version": "1.80.0", - "model": "gpt-4o", - "provider": "openai", - "pricing_source": "cost_map", - "priced_model_key": "openai/gpt-4o", - "multipliers": (Multiplier(name="data_residency", value="eu", factor=Decimal("1.1")),), - "rates": (Rate(measure="input", unit="token", usd_per_unit=Decimal("0.0000025")),), - "usage": (PricedQuantity(measure="input", unit="token", value=_TOKEN.value),), - "input_cost": Decimal("0.000025"), - "total_cost": _SPEND.value, - **changes, - } - ) - - -def test_cost_event_carries_a_snapshot_whose_total_matches_the_spend(): - event = CostEvent( - event_id="cost:attempt", - usage_event_id="outbound:attempt", - occurred_at=_START, - dimensions=(_TEAM,), - charges=(_SPEND,), - snapshot=_snapshot(), - ) - assert event.snapshot is not None - assert event.snapshot.total_cost == _SPEND.value - assert event.snapshot.multipliers[0].factor == Decimal("1.1") - with pytest.raises(ValidationError, match="Snapshot total must equal the authoritative spend"): - CostEvent( - event_id="cost:attempt", - usage_event_id="outbound:attempt", - occurred_at=_START, - dimensions=(_TEAM,), - charges=(_SPEND,), - snapshot=_snapshot(total_cost=Decimal("0.5")), - ) - - -@pytest.mark.parametrize( - ("changes", "message"), - ( - ( - { - "rates": ( - Rate(measure="input", unit="token", usd_per_unit=Decimal(1)), - Rate(measure="input", unit="token", usd_per_unit=Decimal(2)), - ) - }, - "Duplicate rate measures and units", - ), - ({"usage": (_SPEND,)}, "PricedQuantity"), - ({"usage": (_REQUEST,)}, "PricedQuantity"), - ({"pricing_source": "guess"}, "pricing_source"), - ({"multipliers": ({"name": "x", "value": "y", "factor": Decimal(0)},)}, "factor"), - ({"schema_version": 2}, "schema_version"), - ( - {"additional_costs": ({"name": "fee", "amount": Decimal(1)}, {"name": "fee", "amount": Decimal(2)})}, - "Duplicate additional cost names", - ), - ), -) -def test_snapshots_reject_ambiguous_rates_wrong_usage_and_unknown_sources(changes: Mapping[str, object], message: str): - with pytest.raises(ValidationError, match=message): - _snapshot(**changes) - - -@pytest.mark.parametrize( - ("unit", "value"), - [("token", "10"), ("second", "1.25"), ("character", "25"), ("image", "2"), ("request", "1")], -) -def test_snapshot_preserves_matching_rates_and_quantities_for_each_billing_unit(unit: PricingUnit, value: str) -> None: - quantity: Final = PricedQuantity(measure="input", unit=unit, value=Decimal(value)) - rate: Final = Rate(measure="input", unit=unit, usd_per_unit=Decimal("0.003")) - amount: Final = quantity.value * rate.usd_per_unit - snapshot: Final = _snapshot(rates=(rate,), usage=(quantity,), total_cost=amount) - restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json()) - assert restored == snapshot - assert restored.usage[0].value * restored.rates[0].usd_per_unit == amount - - -@pytest.mark.parametrize( - ("usage", "message"), - [ - ((), "Every rate requires"), - (({"measure": "output", "unit": "token", "value": Decimal(10)},), "Every rate requires"), - (({"measure": "input", "unit": "second", "value": Decimal(10)},), "Every rate requires"), - ( - ({"measure": "input", "unit": "token", "value": Decimal(10)},) * 2, - "Duplicate priced quantity", - ), - ], -) -def test_snapshot_rejects_missing_mismatched_or_duplicate_priced_quantities( - usage: tuple[Mapping[str, object], ...], message: str -) -> None: - with pytest.raises(ValidationError, match=message): - _snapshot(usage=usage) - - -@pytest.mark.parametrize("unit", ["token", "character", "image", "request"]) -def test_priced_counts_remain_integral(unit: PricingUnit) -> None: - with pytest.raises(ValidationError, match="integral"): - PricedQuantity(measure="input", unit=unit, value=Decimal("1.25")) - - -@pytest.mark.parametrize( - ("value", "message"), - [ - (Decimal(-1), "greater than or equal"), - (Decimal("NaN"), "finite"), - (Decimal("Infinity"), "finite"), - (1.25, "instance of Decimal"), - ], -) -def test_pricing_values_reject_invalid_amounts(value: Decimal | float, message: str) -> None: - with pytest.raises(ValidationError, match=message): - PricedQuantity.model_validate({"measure": "input", "unit": "second", "value": value}) - with pytest.raises(ValidationError, match=message): - Rate.model_validate({"measure": "input", "unit": "second", "usd_per_unit": value}) - - -@pytest.mark.parametrize("snapshot_fields", [{}, {"snapshot": None}]) -def test_cost_event_cannot_omit_its_derivation(snapshot_fields: Mapping[str, object]) -> None: - with pytest.raises(ValidationError, match="snapshot"): - CostEvent.model_validate( - { - "event_id": "cost", - "usage_event_id": "usage", - "occurred_at": _START, - "dimensions": (), - "charges": (_SPEND,), - **snapshot_fields, - } - ) - - -def test_upstream_charge_snapshot_preserves_reported_cost_without_inventing_rates() -> None: - snapshot: Final = _snapshot( - pricing_source="upstream_header", rates=(), usage=(), upstream_reported_cost=_SPEND.value - ) - restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json()) - assert restored.upstream_reported_cost == restored.total_cost == _SPEND.value - assert restored.rates == restored.usage == () - - -def test_subset_token_counts_round_trip_without_changing_totals() -> None: - metrics: Final = ( - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(10)), - Metric(name=MetricName.REASONING_TOKENS, value=Decimal(3)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(8)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(2)), - ) - event: Final = _outbound(metrics=metrics) - assert OutboundUsageEvent.model_validate_json(event.model_dump_json()).metrics == metrics - - -@pytest.mark.parametrize( - "schema_path", - [ - "schema.prisma", - "litellm/proxy/schema.prisma", - "litellm-proxy-extras/litellm_proxy_extras/schema.prisma", - ], -) -@pytest.mark.parametrize( - ("enum_name", "python_enum"), - [ - ("AccountingDeliveryState", DeliveryState), - ("AccountingDeliveryTarget", DeliveryTarget), - ], -) -def test_prisma_accepts_exactly_the_public_accounting_enum_values( - schema_path: str, enum_name: str, python_enum: type[Enum] -) -> None: - schema: Final = (Path(__file__).resolve().parents[3] / schema_path).read_text() - declaration: Final = re.search(r"enum " + enum_name + r"\s*\{([^}]+)\}", schema) - assert declaration is not None, f"Missing persisted enum {enum_name}" - assert frozenset(declaration[1].split()) == frozenset(member.value for member in python_enum) - - -def test_one_event_carries_every_matched_model_access_group(): - groups = ( - Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="gpt-group"), - Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="claude-group"), - ) - assert _outbound(dimensions=(_TEAM, *groups)).dimensions == (_TEAM, *groups) - with pytest.raises(ValidationError, match="Duplicate dimension values"): - _outbound(dimensions=(groups[0], groups[0])) - - -def test_snapshots_reject_duplicate_multipliers_and_contradictory_token_counts(): - multiplier = Multiplier(name="data_residency", value="eu", factor=Decimal("1.1")) - with pytest.raises(ValidationError, match="Duplicate multiplier names"): - _snapshot(multipliers=(multiplier, multiplier)) - with pytest.raises(ValidationError, match="reasoning_tokens cannot exceed completion_tokens"): - _snapshot( - usage=( - PricedQuantity(measure="input", unit="token", value=_TOKEN.value), - PricedQuantity(measure="completion_tokens", unit="token", value=Decimal(5)), - PricedQuantity(measure="reasoning_tokens", unit="token", value=Decimal(6)), - ) - ) - - -def test_agent_live_spend_is_a_delivery_target(): - assert DeliveryTarget.AGENT_SPEND.value == "LiteLLM_AgentsTable" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 914eb1f2508..4a07c97b709 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -252,6 +252,12 @@ def socket_tempdir(socket_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path: return socket_dir +@pytest.fixture +def python_router_core(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep ``Router`` on ``PythonCore``, for a test that patches a member the Rust core never reads.""" + monkeypatch.setenv("LITELLM_RUST", "0") + + @pytest.fixture(autouse=True, scope="session") def bundled_tiktoken_cache() -> None: importlib.import_module("litellm.litellm_core_utils.default_encoding") diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 1e42e74806d..0e2813e0a0d 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,8 +6,10 @@ import copy import json import re import threading +import uuid from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from concurrent.futures import wait +from datetime import datetime from functools import partial from types import MappingProxyType from typing import Final, TypeAlias @@ -300,6 +302,43 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( assert response.answers["sentiment"].choice == "positive" +@pytest.mark.asyncio +async def test_least_busy_frees_a_decisions_deployment_once_a_proxy_logged_call_finishes( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": {"model": "typesafe/jev-latest", "api_key": "k"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("jev-a", "jev-b") + ], + routing_strategy="least-busy", + ) + + async def call_through_proxy_logging() -> object: + logging_obj, _ = litellm.utils.function_setup( + original_function="adecisions", + rules_obj=litellm.utils.Rules(), + start_time=datetime(2026, 1, 1), + model="jev", + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.adecisions( + model="jev", + input="Is this a defect?", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + litellm_logging_obj=logging_obj, + ) + await _drain_logging_worker() + return response._hidden_params["model_id"] + + assert (await call_through_proxy_logging(), await call_through_proxy_logging()) == ("jev-a", "jev-a") + + def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) diff --git a/tests/unit/integrations/email_alerting/__init__.py b/tests/unit/integrations/email_alerting/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/integrations/email_alerting/test_model_deprecation_email.py b/tests/unit/integrations/email_alerting/test_model_deprecation_email.py new file mode 100644 index 00000000000..54dbe342148 --- /dev/null +++ b/tests/unit/integrations/email_alerting/test_model_deprecation_email.py @@ -0,0 +1,298 @@ +"""Retirement reminder schedule, durable dedupe, ownership scope and safe rendering.""" + +import asyncio +from dataclasses import replace +from datetime import date, datetime, timedelta, timezone +from types import MappingProxyType +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.integrations import model_deprecation_email as module +from litellm.integrations.email_templates.model_deprecation_email import ( + render_model_deprecation_email, +) +from litellm.models.team import LiteLLM_TeamTable +from litellm.models.user import LiteLLM_UserTable +from litellm.proxy._types import LiteLLM_VerificationToken +from litellm.repositories.config_repository import ConfigParam +from litellm.types.integrations.model_deprecation_email import ( + DeprecationEmailMilestone, + ModelDeprecationEmail, +) +from litellm.types.proxy.model_deprecation import ( + ModelDeprecationInfo, + ModelDeprecationResponse, + SuccessorModelSource, +) + +NOW = datetime(2026, 11, 1, tzinfo=timezone.utc) + + +def info(days=30, model="gemini/gemini-2.5-pro"): + return ModelDeprecationInfo( + model_name="research", + litellm_model=model, + deprecation_date=NOW.date() + timedelta(days=days), + days_until_deprecation=days, + status="imminent", + successor_model="gemini/gemini-3.8-flash", + successor_model_source=SuccessorModelSource.MODEL_COST_MAP, + deprecation_source="https://ai.google.dev/gemini-api/docs/deprecations", + ) + + +class MemoryConfig: + def __init__(self): + self.rows = {"general_settings": {"email_settings": {"Model Deprecations": True}}} + + async def get_param(self, key): + return ConfigParam(key, self.rows[key]) if key in self.rows else None + + async def set_param_if_revision(self, key, value, revision): + current = self.rows.get(key, {}).get("revision", 0) + if current != revision: + return False + self.rows[key] = value + return True + + async def delete_params_older_than(self, prefix, before): + return 0 + + +@pytest.fixture() +def sender(monkeypatch): + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example") + instance = module.ModelDeprecationEmailSender(MagicMock()) + instance.config = MemoryConfig() + instance._impacts = AsyncMock( + return_value=MappingProxyType( + {"owner@example.com": module._Impact(key_count=3, requests=12, teams=frozenset({"Research"}))} + ) + ) + send = AsyncMock(return_value=True) + monkeypatch.setattr("litellm.proxy.utils.send_email", send) + return instance, send + + +def schedule(monkeypatch, models): + snapshot = ModelDeprecationResponse( + deprecated=[], imminent=models, upcoming=[], warn_within_days=30, checked_at=NOW + ) + monkeypatch.setattr(module, "collect_model_deprecations", lambda **kwargs: snapshot) + + +@pytest.mark.asyncio +async def test_should_send_only_the_two_milestones_and_keep_receipts_across_restart(sender, monkeypatch): + instance, send = sender + schedule(monkeypatch, [info()]) + assert await instance.run(MagicMock(), now=NOW) == 1 + assert await instance.run(MagicMock(), now=NOW) == 0 + restarted = module.ModelDeprecationEmailSender(MagicMock()) + restarted.config = instance.config + restarted._impacts = instance._impacts + assert await restarted.run(MagicMock(), now=NOW) == 0 + final_day = NOW + timedelta(days=29) + schedule(monkeypatch, [info().model_copy(update={"days_until_deprecation": 1})]) + assert await restarted.run(MagicMock(), now=final_day) == 1 + assert await restarted.run(MagicMock(), now=final_day) == 0 + assert send.await_count == 2 + assert "in 30 days" in send.await_args_list[0].kwargs["subject"] + assert "tomorrow" in send.await_args_list[1].kwargs["subject"] + assert send.await_args_list[0].kwargs["message_id"] != send.await_args_list[1].kwargs["message_id"] + assert "alert("x")', + "deprecation_source": "javascript:alert(1)", + } + ) + email = ModelDeprecationEmail( + model=model, + usage_model=model.litellm_model, + recipient_email="owner@example.com", + milestone=DeprecationEmailMilestone.THIRTY_DAYS, + key_count=3, + requests=12, + team_names=("Research",), + start_date=date(2026, 10, 3), + end_date=NOW.date(), + ) + rendered = render_model_deprecation_email(email, base_url="https://gateway.example") + assert "/public/assets/logos/litellm_logo.png" in rendered.html + assert "litellm-listing.s3.amazonaws.com" not in rendered.html + assert "