Merge pull request #50856 from BerriAI/litellm_backport_43948_stable_1_103_x_devin

fix(proxy-extras): backport the automatic SpendLogs index build (#43948, #44109, #44203) to stable/1.103.x
This commit is contained in:
yuneng-jiang 2026-11-10 09:27:20 -08:00 • committed by GitHub
commit 8f44f66fde
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 4244 additions and 569 deletions

View file

@ -55,6 +55,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_session_id_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_api_key_startTime_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_litellm_call_id_idx";
CREATE TABLE "LiteLLM_SpendLogs" (
LIKE "LiteLLM_SpendLogs_legacy" INCLUDING DEFAULTS INCLUDING GENERATED

View file

@ -0,0 +1,50 @@
import functools
import os
import re
from collections.abc import Callable
from typing import Final
from urllib.parse import unquote, urlsplit
_REDACTED: Final = "REDACTED"
_PASSWORD_QUERY_KEYS: Final = frozenset(("password", "sslpassword"))
@functools.cache
def _secret_shape_redactor() -> Callable[[str], str]:
try:
from litellm._logging import redact_secrets
except ImportError:
return lambda text: text
return redact_secrets
def _url_passwords(url: str) -> frozenset[str]:
try:
parts: Final = urlsplit(url)
except ValueError:
return frozenset()
query_pairs: Final = tuple(pair.partition("=") for pair in parts.query.split("&"))
raw_query_passwords: Final = tuple(
value for key, separator, value in query_pairs if separator and key.lower() in _PASSWORD_QUERY_KEYS
)
raw_passwords: Final = ((parts.password,) if parts.password else ()) + raw_query_passwords
return frozenset(password for password in raw_passwords + tuple(map(unquote, raw_passwords)) if password)
def _configured_database_passwords() -> frozenset[str]:
database_url: Final = os.getenv("DATABASE_URL")
direct_url: Final = os.getenv("DIRECT_URL")
database_passwords: Final = _url_passwords(database_url) if database_url else frozenset()
direct_passwords: Final = _url_passwords(direct_url) if direct_url else frozenset()
return database_passwords | direct_passwords
def _redact_credentials(text: str) -> str:
"""Mask configured database passwords before passing the text to LiteLLM redaction."""
passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True)
alternation: Final = "|".join(re.escape(password) for password in passwords)
password_pattern: Final = (
re.compile(rf"(?<![A-Za-z0-9_])(?:{alternation})(?![A-Za-z0-9_])", re.IGNORECASE) if passwords else None
)
result: Final = password_pattern.sub(_REDACTED, text) if password_pattern is not None else text
return _secret_shape_redactor()(result)

View file

@ -0,0 +1,140 @@
import random
import re
import time
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras._redaction import _redact_credentials
from litellm_proxy_extras.prisma_toolchain import (
MIGRATION_LOCK_TIMEOUT_ENV_VAR,
migration_lock_timeout,
)
MIGRATION_LOCK_KEY: Final = int.from_bytes(b"llm_mig2", "big")
PRISMA_ADVISORY_LOCK_KEY: Final = 72707369
_SERVER_ANSWERED_MARKERS: Final = ("FATAL:", "SSL", "certificate")
if TYPE_CHECKING:
import psycopg
def migration_environment(environment: Mapping[str, str]) -> Mapping[str, str]:
database_url: Final = environment.get("DATABASE_URL")
direct_url: Final = environment.get("DIRECT_URL")
if not database_url or not direct_url:
return environment
schema: Final = next((value for key, value in parse_qsl(urlsplit(database_url).query) if key == "schema"), "public")
direct: Final = urlsplit(direct_url)
parameters: Final = tuple((key, value) for key, value in parse_qsl(direct.query) if key != "schema")
return {
**environment,
"DATABASE_URL": urlunsplit(direct._replace(query=urlencode((*parameters, ("schema", schema))))),
}
@dataclass(frozen=True, slots=True)
class _LockResult:
acquired: bool
def _try_lock(connection: "psycopg.Connection[tuple[object, ...]]", key: int = MIGRATION_LOCK_KEY) -> bool:
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(_LockResult)) as cursor:
row: Final = cursor.execute("SELECT pg_try_advisory_xact_lock(%s) AS acquired", (key,)).fetchone()
return row is not None and row.acquired
def _coordination_failure(database_url: str, exc: "psycopg.Error", *, connecting: bool) -> str:
import psycopg
prefix: Final = "Lost or could not establish v2 migration coordination with the database: "
raw: Final = str(exc)
unquoted: Final = re.sub(r"\"[^\"]*\"|'[^']*'", "", raw)
detail: Final = _redact_credentials(raw).strip()
if (
not connecting
or not isinstance(exc, psycopg.OperationalError)
or any(marker in unquoted for marker in _SERVER_ANSWERED_MARKERS)
):
return prefix + detail
parsed_url: Final = urlsplit(database_url)
try:
port: Final = parsed_url.port
except ValueError:
return prefix + f"P1001: Can't reach database server. {detail}"
host: Final = parsed_url.hostname or "localhost"
return prefix + f"P1001: Can't reach database server at `{host}:{port or 5432}`. {detail}"
@dataclass(frozen=True, slots=True)
class MigrationCoordinator:
connection: "psycopg.Connection[tuple[object, ...]]"
def check_connection(self) -> None:
self.connection.execute("SELECT 1")
def acquire_prisma_lock(self) -> None:
deadline: Final = time.monotonic() + migration_lock_timeout()
while time.monotonic() < deadline:
if _try_lock(self.connection, 72707369):
return
time.sleep(min(random.uniform(0.5, 1.5), max(0.0, deadline - time.monotonic())))
raise RuntimeError(
"Timed out waiting for Prisma's lock to recover migration history. LiteLLM startup has stopped. "
"Another migration or a pooled database session may still hold the lock. Check the database lock holder. "
"When using a transaction pooler, configure DIRECT_URL to reach the same database without the pooler."
)
@contextmanager
def migration_lock(database_url: str) -> Generator[MigrationCoordinator, None, None]:
import psycopg
wait_seconds: Final = migration_lock_timeout()
deadline: Final = time.monotonic() + wait_seconds
try:
connection: Final = psycopg.connect(database_url, connect_timeout=10, autocommit=True)
except psycopg.Error as exc:
raise RuntimeError(_coordination_failure(database_url, exc, connecting=True)) from None
try:
with connection:
coordinator: Final = MigrationCoordinator(connection)
logger.info("Waiting for the v2 migration coordinator lock (up to %ss)", wait_seconds)
while time.monotonic() < deadline:
with connection.transaction():
if _try_lock(connection):
logger.info("Acquired the v2 migration coordinator lock")
yield coordinator
coordinator.check_connection()
return
time.sleep(min(random.uniform(0.5, 1.5), max(0.0, deadline - time.monotonic())))
except psycopg.Error as exc:
raise RuntimeError(_coordination_failure(database_url, exc, connecting=False)) from None
raise RuntimeError(
f"Timed out waiting for another v2 migration resolver after {wait_seconds}s. "
f"Check the running migration or increase {MIGRATION_LOCK_TIMEOUT_ENV_VAR}."
)
@contextmanager
def held_migration_lock(connection: "psycopg.Connection[tuple[object, ...]]") -> Generator[bool, None, None]:
"""A session-level, non-blocking hold of the migration coordinator lock on an autocommit
connection, for DDL that cannot run inside a transaction (`CREATE INDEX CONCURRENTLY`).
Yields whether the lock was acquired; a v2 resolver or another migration job's index build
holding it yields False. Released on exit."""
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(_LockResult)) as cursor:
row: Final = cursor.execute("SELECT pg_try_advisory_lock(%s) AS acquired", (MIGRATION_LOCK_KEY,)).fetchone()
acquired: Final = row is not None and row.acquired
try:
yield acquired
finally:
if acquired:
connection.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_KEY,))

View file

@ -0,0 +1,204 @@
import hashlib
import re
import subprocess
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Final
from uuid import uuid4
from litellm_proxy_extras import prisma_toolchain
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.migration_lock import MigrationCoordinator
if TYPE_CHECKING:
import psycopg
@dataclass(frozen=True, slots=True)
class MigrationProgress:
checksum: str
applied_steps_count: int
logs: str
id: str = ""
finished: bool = False
def confirms_completion(self, script: bytes) -> bool:
return (
self.applied_steps_count == 1
and not self.logs.strip()
and self.checksum == hashlib.sha256(script).hexdigest()
)
def _migration_records(
connection: "psycopg.Connection[tuple[object, ...]]", schema: str, migration: Path
) -> tuple[MigrationProgress, ...]:
from psycopg import sql
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(MigrationProgress)) as cursor:
records: Final = cursor.execute(
sql.SQL(
"SELECT id, checksum, applied_steps_count, coalesce(logs, '') AS logs, "
"finished_at IS NOT NULL AS finished FROM {} "
"WHERE migration_name = %s AND rolled_back_at IS NULL"
).format(sql.Identifier(schema, "_prisma_migrations")),
(migration.parent.name,),
).fetchall()
return tuple(records)
def recover_completed_migration(coordinator: MigrationCoordinator, schema: str, migration: Path) -> bool:
"""Finish a proven successful row without erasing its durable completion evidence.
The caller commits this checkpoint before running another Prisma command.
"""
from psycopg import sql
coordinator.acquire_prisma_lock()
records: Final = _migration_records(coordinator.connection, schema, migration)
unfinished: Final = tuple(record for record in records if not record.finished)
script: Final = migration.read_bytes()
if not unfinished:
return any(record.checksum == hashlib.sha256(script).hexdigest() for record in records)
if len(unfinished) != 1 or not unfinished[0].confirms_completion(script):
return False
progress: Final = unfinished[0]
result: Final = coordinator.connection.execute(
sql.SQL(
"UPDATE {} SET finished_at = current_timestamp "
"WHERE id = %s AND checksum = %s AND applied_steps_count = 1 "
"AND finished_at IS NULL AND rolled_back_at IS NULL AND coalesce(logs, '') = %s"
).format(sql.Identifier(schema, "_prisma_migrations")),
(progress.id, progress.checksum, progress.logs),
)
if result.rowcount != 1:
raise RuntimeError("Could not complete the confirmed migration history row; retry startup.")
logger.info("Completed migration %s using its successful SQL step and matching checksum", migration.parent.name)
return True
def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
return tuple(
(path.parent.name, hashlib.sha256(path.read_bytes()).hexdigest())
for path in sorted((directory / "migrations").glob("*/migration.sql"))
)
def baseline_current_schema(
coordinator: MigrationCoordinator,
schema: str,
migrations_dir: Path,
prisma_command: str,
prisma_env: Mapping[str, str],
) -> None:
from psycopg import sql
packaged_dir: Final = Path(__file__).parent
migrations: Final = migration_files(migrations_dir)
if (
not migrations
or migrations != migration_files(packaged_dir)
or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
):
raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
coordinator.acquire_prisma_lock()
existing: Final = coordinator.connection.execute(
"SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(coordinator.connection),)
).fetchone()
if existing is not None and existing[0] is not None:
return
try:
prisma_toolchain.run_prisma(
(
prisma_command,
"migrate",
"diff",
"--from-schema-datasource",
str(migrations_dir / "schema.prisma"),
"--to-schema-datamodel",
str(migrations_dir / "schema.prisma"),
"--exit-code",
),
timeout=prisma_toolchain.prisma_command_timeout(),
env=prisma_env,
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
raise RuntimeError(
"Cannot automatically baseline this database: its schema has not been verified to match this build. "
"Establish the existing migration history before retrying. No schema reconciliation was performed. "
"If using a transaction pooler, configure DIRECT_URL to reach the same database without the pooler. "
f"Schema verification detail: {exc.stderr}"
) from exc
coordinator.check_connection()
ledger: Final = sql.Identifier(schema, "_prisma_migrations")
coordinator.connection.execute(
sql.SQL(
"CREATE TABLE {} (id varchar(36) PRIMARY KEY NOT NULL, checksum varchar(64) NOT NULL, "
"finished_at timestamptz, migration_name varchar(255) NOT NULL, logs text, rolled_back_at timestamptz, "
"started_at timestamptz NOT NULL DEFAULT now(), applied_steps_count integer NOT NULL DEFAULT 0)"
).format(ledger)
)
with coordinator.connection.cursor() as cursor:
cursor.executemany(
sql.SQL(
"INSERT INTO {} (id, checksum, migration_name, logs, started_at, finished_at) "
"VALUES (%s, %s, %s, '', current_timestamp, current_timestamp)"
).format(ledger),
tuple((str(uuid4()), checksum, name) for name, checksum in migrations),
)
logger.warning(
"Legacy migration history was missing. The existing Prisma schema matches this build; "
"adopted %s packaged migrations as a baseline. No schema changes were applied, and "
"historical data backfills were not replayed or verified. Continuing startup; "
"review any feature-specific backfill requirements.",
len(migrations),
)
_LINE_COMMENT_RE: Final = re.compile(r"--[^\n]*")
_BLOCK_COMMENT_RE: Final = re.compile(r"/\*.*?\*/", re.DOTALL)
_NO_OP_STATEMENT_RE: Final = re.compile(r"^\s*SELECT\s+1\s*$", re.IGNORECASE)
def is_inert_migration(script: str) -> bool:
"""Whether a migration file changes nothing: only comments and `SELECT 1`, so
applying it can neither repeat nor skip a database change."""
stripped: Final = _LINE_COMMENT_RE.sub("", _BLOCK_COMMENT_RE.sub("", script))
return all(not part.strip() or _NO_OP_STATEMENT_RE.match(part) for part in stripped.split(";"))
def roll_back_failed_inert_migration(coordinator: MigrationCoordinator, schema: str, migration: Path) -> bool:
"""Roll back the failed ledger row of a migration whose file in this build is inert,
so `migrate deploy` applies the inert file on its next pass. The row records an
earlier build's attempt at SQL this build no longer ships (an index now built by the
migration job), so no database change can be repeated or skipped by replaying
the empty file. The caller commits this checkpoint before the next Prisma command.
"""
from psycopg import sql
if not is_inert_migration(migration.read_text(encoding="utf-8")):
return False
coordinator.acquire_prisma_lock()
records: Final = _migration_records(coordinator.connection, schema, migration)
unfinished: Final = tuple(record for record in records if not record.finished)
if len(unfinished) != 1:
return False
result: Final = coordinator.connection.execute(
sql.SQL(
"UPDATE {} SET rolled_back_at = current_timestamp "
"WHERE id = %s AND finished_at IS NULL AND rolled_back_at IS NULL"
).format(sql.Identifier(schema, "_prisma_migrations")),
(unfinished[0].id,),
)
if result.rowcount != 1:
raise RuntimeError("Could not roll back the failed inert migration history row; rerun the database setup.")
logger.info(
"Rolled back the failed history row of %s: this build ships it as an inert migration, "
"its index is built by the migration job",
migration.parent.name,
)
return True

View file

@ -1 +1,6 @@
-- The (api_key, startTime) index on LiteLLM_SpendLogs is built after migrate deploy,
-- through litellm_proxy_extras/request_log_indexes.py: concurrently on a plain table and
-- per partition on a partitioned one. The migration job builds it; a serving proxy that
-- ran the migrations itself builds it in the background once it serves. A migration
-- cannot do either without blocking spend-log writes or failing on a partitioned table.
SELECT 1;

View file

@ -1 +1,6 @@
-- The litellm_call_id index on LiteLLM_SpendLogs is built after migrate deploy, through
-- litellm_proxy_extras/request_log_indexes.py: concurrently on a plain table and per
-- partition on a partitioned one. The migration job builds it; a serving proxy that ran
-- the migrations itself builds it in the background once it serves. Postgres refuses
-- CREATE INDEX CONCURRENTLY on a partitioned parent, so this migration no longer runs it.
SELECT 1;

View file

@ -59,6 +59,7 @@ except ImportError:
PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT"
MIGRATION_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_LOCK_TIMEOUT"
NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
@ -106,6 +107,10 @@ def prisma_command_timeout() -> float:
)
def migration_lock_timeout() -> float:
return _timeout_from_env(MIGRATION_LOCK_TIMEOUT_ENV_VAR, 600.0)
def prisma_bootstrap_timeout() -> float:
"""Seconds the one-time Node toolchain install may run for."""
return _timeout_from_env(

View file

@ -0,0 +1,480 @@
"""The request-log indexes built after `prisma migrate deploy` instead of by a migration:
by the migration job, or by a serving proxy that ran the migrations itself (in the
background, once it serves).
A migration cannot build them: a plain `CREATE INDEX` blocks spend-log inserts for the
whole build, and `CREATE INDEX CONCURRENTLY` is refused on a partitioned parent
(db_scripts/partition_spend_logs.sql). `REQUEST_LOG_INDEXES` is the one list to extend;
names match what Prisma derives from the `@@index` declarations in schema.prisma, so an
index a database already has is recognized and never rebuilt.
"""
import hashlib
import random
import re
import time
from collections.abc import Callable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.migration_lock import held_migration_lock
if TYPE_CHECKING:
import psycopg
from psycopg import sql
@dataclass(frozen=True, slots=True)
class RequestLogIndex:
"""One index the migration job owns: the table, the exact Prisma index name and the
column list as it would be written after `ON <table>`."""
table: str
name: str
definition: str
@property
def columns(self) -> tuple[str, ...]:
return tuple(re.findall(r'"([^"]+)"', self.definition))
def partition_index_name(self, partition: str) -> str:
"""The child index name for one partition, built the way Postgres names the
children of a partitioned index, and kept within the 63 byte identifier limit."""
name: Final = f"{partition}_{self.name.removeprefix(f'{self.table}_')}"
if len(name.encode()) <= _IDENTIFIER_MAX_BYTES:
return name
digest: Final = hashlib.sha256(name.encode()).hexdigest()[:_DIGEST_LENGTH]
budget: Final = _IDENTIFIER_MAX_BYTES - _DIGEST_LENGTH - 1
kept: Final = next(name[:length] for length in range(len(name), 0, -1) if len(name[:length].encode()) <= budget)
return f"{kept}_{digest}"
REQUEST_LOG_INDEXES: Final = (
RequestLogIndex("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", '("api_key", "startTime")'),
RequestLogIndex("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_litellm_call_id_idx", '("litellm_call_id")'),
)
_IDENTIFIER_MAX_BYTES: Final = 63
_DDL_LOCK_TIMEOUT: Final = "200ms"
_DDL_LOCK_ATTEMPTS: Final = 10
_DDL_RETRY_BASE_SECONDS: Final = 0.25
_DDL_RETRY_MAX_SECONDS: Final = 8.0
_LOCK_HANDOVER_SECONDS: Final = 2.0
_DIGEST_LENGTH: Final = 8
_CREATE_INDEX_STATEMENT: Final = re.compile(
r'^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?"(?P<index>[^"]+)"\s+ON\b',
re.IGNORECASE,
)
_TABLE_KIND_SQL: Final = "SELECT c.relkind = 'p' AS partitioned FROM pg_class c WHERE c.oid = to_regclass(%s)"
_REQUEST_LOG_TABLES_SQL: Final = (
"WITH RECURSIVE tables AS ("
"SELECT c.oid, c.relname, c.relnamespace FROM pg_class c "
"JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = %s AND c.relname = ANY(%s) "
"UNION SELECT c.oid, c.relname, c.relnamespace FROM tables p "
"JOIN pg_inherits h ON h.inhparent = p.oid JOIN pg_class c ON c.oid = h.inhrelid"
") SELECT t.relname FROM tables t JOIN pg_namespace n ON n.oid = t.relnamespace WHERE n.nspname = %s"
)
_CHILDREN_WITHOUT_THE_INDEX_SQL: Final = (
"SELECT child.relname AS name, n.nspname AS schema, child.relkind = 'p' AS partitioned "
"FROM pg_inherits i JOIN pg_class child ON child.oid = i.inhrelid "
"JOIN pg_namespace n ON n.oid = child.relnamespace "
"WHERE i.inhparent = to_regclass(%s) AND NOT EXISTS ("
"SELECT 1 FROM pg_inherits attached JOIN pg_index x ON x.indexrelid = attached.inhrelid "
"WHERE attached.inhparent = to_regclass(%s) AND x.indrelid = child.oid) "
"ORDER BY child.relname"
)
_EQUIVALENT_INDEXES_SQL: Final = (
"SELECT i.relname AS name, x.indisvalid AS valid "
"FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid JOIN pg_am am ON am.oid = i.relam "
"WHERE x.indrelid = to_regclass(%s) AND i.relname <> %s AND am.amname = 'btree' AND NOT x.indisunique "
"AND x.indexprs IS NULL AND x.indpred IS NULL AND cardinality(x.indclass::oid[]) = x.indnatts "
"AND NOT EXISTS (SELECT 1 FROM unnest(x.indoption::int2[]) o WHERE o <> 0) "
"AND NOT EXISTS (SELECT 1 FROM unnest(x.indclass::oid[]) c JOIN pg_opclass oc ON oc.oid = c WHERE NOT oc.opcdefault) "
"AND NOT EXISTS (SELECT 1 FROM unnest(x.indcollation::oid[]) WITH ORDINALITY c(coll, ord) "
"JOIN unnest(x.indkey::int2[]) WITH ORDINALITY k(attnum, ord) ON k.ord = c.ord "
"JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum "
"WHERE c.coll <> 0 AND c.coll <> a.attcollation) "
"AND (SELECT array_agg(a.attname::text ORDER BY k.ord) FROM unnest(x.indkey::int2[]) WITH ORDINALITY k(attnum, ord) "
"JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum) = %s::text[] "
"AND NOT EXISTS (SELECT 1 FROM pg_inherits WHERE inhrelid = x.indexrelid) "
"ORDER BY x.indisvalid DESC, i.relname"
)
_INDEX_STATE_SQL: Final = (
'SELECT x.indisvalid AS valid, t.relname AS "table" '
"FROM pg_index x JOIN pg_class t ON t.oid = x.indrelid WHERE x.indexrelid = to_regclass(%s)"
)
@dataclass(frozen=True, slots=True)
class _Relation:
name: str
schema: str
partitioned: bool
@dataclass(frozen=True, slots=True)
class _IndexState:
valid: bool
table: str
@dataclass(frozen=True, slots=True)
class _EquivalentIndex:
name: str
valid: bool
@dataclass(frozen=True, slots=True)
class _TableKind:
partitioned: bool
def filter_request_log_index_diff(diff_sql: str, indexes: tuple[RequestLogIndex, ...] = REQUEST_LOG_INDEXES) -> str:
"""The `prisma migrate diff` script without the statements that create a migration-job-owned
index, which the schema declares and the migrations deliberately do not build."""
names: Final = frozenset(index.name for index in indexes)
statements: Final = diff_sql.split(";")
kept: Final = tuple(statement for statement in statements if not _creates_one_of(statement, names))
return ";".join(kept) if any(part.strip() for part in kept) else ""
def _creates_one_of(statement: str, names: frozenset[str]) -> bool:
match: Final = _CREATE_INDEX_STATEMENT.match(_without_comments(statement))
return match is not None and match["index"] in names
def _without_comments(statement: str) -> str:
return "\n".join(line for line in statement.splitlines() if not line.lstrip().startswith("--"))
def _connect(database_url: str) -> "psycopg.Connection[tuple[object, ...]]":
import psycopg
return psycopg.connect(database_url, connect_timeout=10, autocommit=True)
def ensure_request_log_indexes(
database_url: str,
schema: str,
indexes: tuple[RequestLogIndex, ...] = REQUEST_LOG_INDEXES,
connect: "Callable[[str], psycopg.Connection[tuple[object, ...]]]" = _connect,
) -> bool:
"""Build every listed index that is missing or invalid. Each build step runs under
the migration coordinator lock, held per statement so a resolver booting on another
replica gets in between partitions rather than waiting for the whole table. Any
failure is logged and left for the next index build; the result says whether
every index ended up valid. Never raises."""
import psycopg
try:
with connect(database_url) as connection:
connection.execute("SET statement_timeout = 0")
results: Final = tuple(_ensure_index(connection, schema, index) for index in indexes)
except psycopg.Error as exc:
logger.warning("Could not build the request-log indexes, leaving them for the next index build: %s", exc)
return False
if not all(results):
logger.warning("Some request-log indexes are not in place yet, leaving them for the next index build")
return False
logger.info("Request-log indexes are all in place")
return True
def request_log_index_names(connection: "psycopg.Connection[tuple[object, ...]]", schema: str) -> frozenset[str]:
tables: Final = connection.execute(
_REQUEST_LOG_TABLES_SQL, (schema, sorted(frozenset(index.table for index in REQUEST_LOG_INDEXES)), schema)
).fetchall()
return frozenset(index.name for index in REQUEST_LOG_INDEXES) | frozenset(
index.partition_index_name(str(table[0])) for table in tables for index in REQUEST_LOG_INDEXES
)
def _under_migration_lock(connection: "psycopg.Connection[tuple[object, ...]]", step: Callable[[], bool]) -> bool:
with held_migration_lock(connection) as held:
if not held:
logger.info(
"Another process holds the migration lock, leaving the request-log indexes to the next index build"
)
return False
return step()
def _with_bounded_lock(
connection: "psycopg.Connection[tuple[object, ...]]", step: Callable[[], bool], what: str
) -> bool:
"""Run `step` under the migration lock with a short lock_timeout, so a DDL statement that has to wait for open
transactions holds new writes back for at most that long; retry with capped exponential backoff, holding the
migration lock per attempt only and releasing it while sleeping. False when another process holds the migration
lock or every attempt timed out."""
import psycopg
from psycopg import sql
for attempt in range(_DDL_LOCK_ATTEMPTS):
if attempt:
time.sleep(min(_DDL_RETRY_MAX_SECONDS, _DDL_RETRY_BASE_SECONDS * 2.0**attempt) * random.uniform(0.5, 1.0))
connection.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(_DDL_LOCK_TIMEOUT)))
try:
return _under_migration_lock(connection, step)
except psycopg.errors.LockNotAvailable:
logger.info("Waiting for open transactions before %s", what)
finally:
connection.execute("SET lock_timeout = 0")
logger.warning(
"Could not get the lock for %s without holding writes back, leaving it for the next index build", what
)
return False
def _ensure_index(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: RequestLogIndex) -> bool:
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(_TableKind)) as cursor:
table: Final = cursor.execute(_TABLE_KIND_SQL, (_regclass_name(connection, schema, index.table),)).fetchone()
if table is None:
logger.info("Table %s does not exist yet, skipping index %s", index.table, index.name)
return True
if table.partitioned:
return build_index_on_partitioned_table(connection, schema, index)
return _build_leaf_index(connection, schema, index.table, index.name, index)
def _regclass_name(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, name: str) -> str:
from psycopg import sql
return sql.Identifier(schema, name).as_string(connection)
def _create_index_statement(
connection: "psycopg.Connection[tuple[object, ...]]", prefix: "sql.Composed", definition: str
) -> bytes:
return (prefix.as_string(connection) + definition).encode()
def _index_state(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: str) -> "_IndexState | None":
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(_IndexState)) as cursor:
return cursor.execute(_INDEX_STATE_SQL, (_regclass_name(connection, schema, index),)).fetchone()
def _equivalent_indexes(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
table: str,
name: str,
index: RequestLogIndex,
) -> tuple[_EquivalentIndex, ...]:
"""The indexes on `table` other than `name` with the same definition: default btree
over the same columns in the same order, no expression, predicate, DESC or custom
opclass or collation, and not attached under a partitioned index. Valid ones first."""
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(_EquivalentIndex)) as cursor:
return tuple(
cursor.execute(
_EQUIVALENT_INDEXES_SQL, (_regclass_name(connection, schema, table), name, list(index.columns))
).fetchall()
)
def _adopt_equivalent_index(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
table: str,
name: str,
index: RequestLogIndex,
) -> bool:
"""Rename a valid index of the same definition under another name (an operator's
hand-built copy, say) to the name this code expects, instead of building a second
one. RENAME on an index is a catalog change that lets writes through."""
from psycopg import sql
equivalent: Final = next(
(found for found in _equivalent_indexes(connection, schema, table, name, index) if found.valid), None
)
if equivalent is None:
return False
logger.info(
"Renaming the equivalent index %s on %s to %s instead of building a second one", equivalent.name, table, name
)
connection.execute(
sql.SQL("ALTER INDEX {} RENAME TO {}").format(sql.Identifier(schema, equivalent.name), sql.Identifier(name))
)
return True
def _report_second_copies(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
table: str,
name: str,
index: RequestLogIndex,
concurrently: bool,
) -> None:
"""Log every other index of the same definition with the statement that removes it.
Dropping is the operator's call: a second copy costs writes and disk, never results."""
from psycopg import sql
drop: Final = "DROP INDEX CONCURRENTLY" if concurrently else "DROP INDEX"
for copy in _equivalent_indexes(connection, schema, table, name, index):
logger.warning(
"Index %s on %s is a second copy of %s and only costs writes and disk; remove it with: %s %s",
copy.name,
table,
name,
drop,
sql.Identifier(schema, copy.name).as_string(connection),
)
def _children_without_the_index(
connection: "psycopg.Connection[tuple[object, ...]]", schema: str, table: str, index: str
) -> tuple[_Relation, ...]:
from psycopg.rows import class_row
with connection.cursor(row_factory=class_row(_Relation)) as cursor:
return tuple(
cursor.execute(
_CHILDREN_WITHOUT_THE_INDEX_SQL,
(_regclass_name(connection, schema, table), _regclass_name(connection, schema, index)),
).fetchall()
)
def _build_leaf_index(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
table: str,
name: str,
index: RequestLogIndex,
) -> bool:
"""Build one plain table's or partition's index with CONCURRENTLY so writes keep
flowing. The catalog is read under the migration lock, so a replica that saw an
invalid index before the lock finds the valid one another replica just built and
leaves it. An invalid index left by an interrupted build is dropped and rebuilt; a
valid index of the same definition under another name is renamed rather than
duplicated; an index of that name on another table is a collision this code will
not touch."""
from psycopg import sql
def build() -> bool:
existing: Final = _index_state(connection, schema, name)
if existing is not None and existing.table != table:
logger.warning(
"Index %s already exists on %s rather than %s, leaving it alone", name, existing.table, table
)
return False
if existing is not None and existing.valid:
return True
if existing is not None:
logger.info("Dropping the invalid index %s left by an interrupted build on %s", name, table)
connection.execute(sql.SQL("DROP INDEX CONCURRENTLY {}").format(sql.Identifier(schema, name)))
elif _adopt_equivalent_index(connection, schema, table, name, index):
return True
logger.info("Building index %s on %s concurrently", name, table)
prefix: Final = sql.SQL("CREATE INDEX CONCURRENTLY IF NOT EXISTS {} ON {} ").format(
sql.Identifier(name), sql.Identifier(schema, table)
)
connection.execute(_create_index_statement(connection, prefix, index.definition))
built: Final = _index_state(connection, schema, name)
return built is not None and built.valid
current: Final = _index_state(connection, schema, name)
if current is None or not current.valid or current.table != table:
if not _under_migration_lock(connection, build):
return False
time.sleep(_LOCK_HANDOVER_SECONDS)
_report_second_copies(connection, schema, table, name, index, concurrently=True)
return True
def build_index_on_partitioned_table(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
index: RequestLogIndex,
table: "str | None" = None,
name: "str | None" = None,
) -> bool:
"""Build the index the way Postgres allows on a partitioned parent: a metadata-only
parent index ON ONLY the parent, one CONCURRENTLY build per partition, and ATTACH
PARTITION for each child. Partitions that are themselves partitioned get the same
treatment one level down. Every step checks the catalog before acting, so an
interrupted run resumes where it stopped and a second run finds nothing to do; a
parent or child index of the same definition under another name is renamed and
used rather than duplicated. The connection must be in autocommit mode. True when
the parent index ends up valid."""
parent_table: Final = index.table if table is None else table
parent_index: Final = index.name if name is None else name
existing: Final = _index_state(connection, schema, parent_index)
if existing is not None and existing.table != parent_table:
logger.warning(
"Index %s already exists on %s rather than %s, leaving it alone", parent_index, existing.table, parent_table
)
return False
if existing is None and not _with_bounded_lock(
connection,
lambda: (
_adopt_equivalent_index(connection, schema, parent_table, parent_index, index)
or _create_parent_index(connection, schema, parent_index, parent_table, index)
),
f"creating the parent index {parent_index}",
):
return False
children: Final = _children_without_the_index(connection, schema, parent_table, parent_index)
if not all(_attach_child_index(connection, schema, parent_index, child, index) for child in children):
return False
final: Final = _index_state(connection, schema, parent_index)
if final is None or not final.valid:
return False
_report_second_copies(connection, schema, parent_table, parent_index, index, concurrently=False)
return True
def _create_parent_index(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
name: str,
table: str,
index: RequestLogIndex,
) -> bool:
"""Create the metadata-only parent index. The caller bounds Postgres's SHARE lock wait on the parent."""
from psycopg import sql
prefix: Final = sql.SQL("CREATE INDEX IF NOT EXISTS {} ON ONLY {} ").format(
sql.Identifier(name), sql.Identifier(schema, table)
)
statement: Final = _create_index_statement(connection, prefix, index.definition)
connection.execute(statement)
return True
def _attach_child_index(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
parent_index: str,
child: _Relation,
index: RequestLogIndex,
) -> bool:
from psycopg import sql
child_index: Final = index.partition_index_name(child.name)
built: Final = (
build_index_on_partitioned_table(connection, child.schema, index, child.name, child_index)
if child.partitioned
else _build_leaf_index(connection, child.schema, child.name, child_index, index)
)
if not built:
return False
def attach() -> bool:
connection.execute(
sql.SQL("ALTER INDEX {} ATTACH PARTITION {}").format(
sql.Identifier(schema, parent_index), sql.Identifier(child.schema, child_index)
)
)
logger.info("Attached index %s on partition %s to %s", child_index, child.name, parent_index)
return True
return _with_bounded_lock(connection, attach, f"attaching {child_index}")

View file

@ -5,13 +5,16 @@ import re
import shutil
import subprocess
import tempfile
import threading
import time
from collections.abc import Callable
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Final, Optional
from litellm_proxy_extras import prisma_toolchain
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.migration_lock import held_migration_lock
from litellm_proxy_extras.prisma_toolchain import (
PRISMA_COMMAND_TIMEOUT_ENV_VAR,
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
@ -23,6 +26,11 @@ from litellm_proxy_extras.replica_identity import (
REPLICA_IDENTITY_FULL_ENV_VAR,
apply_replica_identity_full,
)
from litellm_proxy_extras.request_log_indexes import (
ensure_request_log_indexes,
filter_request_log_index_diff,
request_log_index_names,
)
if TYPE_CHECKING:
import psycopg
@ -74,19 +82,31 @@ class _InvalidIndex:
table_size: str
MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
LIBPQ_URL_PARAMS: Final = frozenset(
{
"sslmode",
"sslcert",
"sslkey",
"sslrootcert",
"sslpassword",
"application_name",
"connect_timeout",
"client_encoding",
"options",
"service",
"gssencmode",
"krbsrvname",
"target_session_attrs",
}
)
@dataclass(frozen=True)
class _MigrateAttemptBudget:
"""Retries left, and the recoveries already run.
A recovery that lands something new costs nothing, so a database full of
objects `prisma db push` created works through them one per pass. Anything
that made no progress spends an attempt, so a stuck run still gives up.
"""
"""Independent bounds for failed attempts and Prisma lock contention."""
attempts_left: int
recoveries: frozenset[str] = frozenset()
contention_seconds_left: float = 600.0
@property
def exhausted(self) -> bool:
@ -99,10 +119,14 @@ class _MigrateAttemptBudget:
def spend(self) -> "_MigrateAttemptBudget":
return replace(self, attempts_left=self.attempts_left - 1)
def after_recovery(self, recovery: str) -> "_MigrateAttemptBudget":
if recovery in self.recoveries:
return self.spend()
return replace(self, recoveries=self.recoveries | {recovery})
def after_contention(self, elapsed: float) -> "_MigrateAttemptBudget":
remaining: Final = self.contention_seconds_left - elapsed
if remaining <= 0:
raise RuntimeError(
"Timed out waiting for Prisma's migration advisory lock. Check the running migration "
"or increase LITELLM_MIGRATION_LOCK_TIMEOUT."
)
return replace(self, contention_seconds_left=remaining)
_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
@ -115,17 +139,6 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile(
re.IGNORECASE,
)
HAND_BUILT_SPEND_LOGS_INDEXES: Final = frozenset(
{
"LiteLLM_SpendLogs_api_key_startTime_idx",
"LiteLLM_SpendLogs_litellm_call_id_idx",
}
)
_CREATE_INDEX_RE: Final = re.compile(
r'^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?"(?P<index>[^"]+)"\s+ON\b',
re.IGNORECASE,
)
PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
"LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
"so its primary key must include the partition key (\"startTime\"). `prisma db push` "
@ -176,21 +189,6 @@ def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
return "".join(f"{statement};\n\n" for statement in kept)
def filter_hand_built_spend_logs_index_diff(diff_sql: str) -> str:
"""The `prisma migrate diff` script without the statements that create the
LiteLLM_SpendLogs indexes schema.prisma declares but the migrations deliberately
leave for operators to build online."""
kept: Final = tuple(
statement for statement in diff_sql.split(";") if not _creates_hand_built_spend_logs_index(statement)
)
return ";".join(kept) if any(part.strip() for part in kept) else ""
def _creates_hand_built_spend_logs_index(statement: str) -> bool:
match: Final = _CREATE_INDEX_RE.match(_without_sql_comments(statement))
return match is not None and match["index"] in HAND_BUILT_SPEND_LOGS_INDEXES
def _migration_timestamp(name: str) -> int:
"""Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@ -359,9 +357,8 @@ class ProxyExtrasDBManager:
pass
@staticmethod
def _failed_migration_logs(migration_name: str) -> Optional[str]:
"""Return failed migration logs, or None if the ledger is unavailable."""
database_url = os.getenv("DATABASE_URL")
def _read_migration_ledger(query: str, params: tuple[str, ...]) -> "tuple[object, ...] | None":
database_url: Final = os.getenv("DATABASE_URL")
if not database_url:
return None
@ -370,28 +367,37 @@ class ProxyExtrasDBManager:
except ImportError:
return None
cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
ledger_table = psycopg.sql.SQL("{}.{}").format(
psycopg.sql.Identifier(
ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
),
cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
ledger_table: Final = psycopg.sql.SQL("{}.{}").format(
psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"),
psycopg.sql.Identifier("_prisma_migrations"),
)
try:
with psycopg.connect(
cleaned_url, connect_timeout=10, autocommit=True
) as conn:
row = conn.execute(
psycopg.sql.SQL(
"SELECT logs FROM {} "
"WHERE migration_name = %s AND finished_at IS NULL "
"AND rolled_back_at IS NULL"
).format(ledger_table),
(migration_name,),
).fetchone()
with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
row: Final = conn.execute(psycopg.sql.SQL(query).format(ledger_table), params).fetchone()
except (psycopg.OperationalError, psycopg.DatabaseError):
return None
return (row[0] or "") if row else ""
return tuple(row) if row is not None else ()
@staticmethod
def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
row: Final = ProxyExtrasDBManager._read_migration_ledger(
"SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
"AND finished_at IS NULL AND rolled_back_at IS NULL",
(migration_name, started_at),
)
if row is None:
return None
return row[0] if row and isinstance(row[0], str) else ""
@staticmethod
def _failed_migration_recovered(migration_name: str, started_at: str) -> bool:
row: Final = ProxyExtrasDBManager._read_migration_ledger(
"SELECT 1 FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
"AND (finished_at IS NOT NULL OR rolled_back_at IS NOT NULL)",
(migration_name, started_at),
)
return bool(row)
@staticmethod
def _resolve_specific_migration(migration_name: str):
@ -460,12 +466,13 @@ class ProxyExtrasDBManager:
return False
@staticmethod
def _filter_drift_script(diff_sql: str) -> str:
"""The drift script without the hand-built SpendLogs indexes and, when
LiteLLM_SpendLogs is partitioned, without its primary-key rewrite and
partitioning artifacts."""
without_indexes: Final = filter_hand_built_spend_logs_index_diff(diff_sql)
if not ProxyExtrasDBManager.spend_logs_is_partitioned():
def _filter_migration_job_owned_drift(diff_sql: str, partitioned: bool | None = None) -> str:
"""The drift script without the indexes the migration job builds (the schema
declares them, the migrations deliberately do not) and, when LiteLLM_SpendLogs
is partitioned, without its primary-key rewrite and partitioning artifacts."""
without_indexes: Final = filter_request_log_index_diff(diff_sql)
is_partitioned: Final = ProxyExtrasDBManager.spend_logs_is_partitioned() if partitioned is None else partitioned
if not is_partitioned:
return without_indexes
logger.info(
"LiteLLM_SpendLogs is partitioned; removed its primary-key "
@ -553,7 +560,7 @@ class ProxyExtrasDBManager:
return
logger.info(f"Migration diff created at {diff_sql_path}")
filtered_sql: Final = ProxyExtrasDBManager._filter_drift_script(diff_sql_path.read_text())
filtered_sql: Final = ProxyExtrasDBManager._filter_migration_job_owned_drift(diff_sql_path.read_text())
diff_sql_path.write_text(filtered_sql)
if not filtered_sql.strip():
logger.info("Drift script is empty after filtering; nothing to apply")
@ -681,30 +688,50 @@ class ProxyExtrasDBManager:
@staticmethod
def _strip_prisma_query_params(url: str) -> str:
"""Remove Prisma-specific query params (connection_limit, pool_timeout,
schema, etc.) from DATABASE_URL so psycopg can parse it."""
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
"""Rewrite a Prisma-dialect URL for libpq: drop the Prisma-only params
(connection_limit, pool_timeout, schema, pgbouncer, sslaccept, ...) and
translate Prisma's TLS params back, since libpq reads ``sslcert`` as a
client certificate where Prisma reads it as the CA."""
from urllib.parse import parse_qsl, quote, unquote, urlencode, urlparse, urlunparse
parsed = urlparse(url)
parsed: Final = urlparse(url)
netloc: Final = parsed.netloc
userinfo, separator, hostport = netloc.rpartition("@")
user, password_separator, password = userinfo.partition(":")
canonical_userinfo: Final = quote(unquote(user), safe="") + (
password_separator + quote(unquote(password), safe="") if password_separator else ""
)
canonical_netloc: Final = f"{canonical_userinfo}{separator}{hostport}" if separator else netloc
if not parsed.query:
return url
libpq_params = {
"sslmode",
"sslcert",
"sslkey",
"sslrootcert",
"sslpassword",
"application_name",
"connect_timeout",
"client_encoding",
"options",
"service",
"gssencmode",
"krbsrvname",
"target_session_attrs",
}
kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params]
return urlunparse(parsed._replace(query=urlencode(kept, quote_via=quote)))
return url if canonical_netloc == netloc else urlunparse(parsed._replace(netloc=canonical_netloc))
pairs: Final = tuple(parse_qsl(parsed.query))
kept: Final = tuple((k, v) for k, v in pairs if k in LIBPQ_URL_PARAMS)
sslaccept: Final = next((v for k, v in pairs if k == "sslaccept"), None)
libpq_pairs: Final = ProxyExtrasDBManager._libpq_tls_params(kept, sslaccept)
return urlunparse(parsed._replace(netloc=canonical_netloc, query=urlencode(libpq_pairs, quote_via=quote)))
@staticmethod
def _libpq_tls_params(
pairs: "tuple[tuple[str, str], ...]", sslaccept: "str | None"
) -> "tuple[tuple[str, str], ...]":
"""Undo ``translate_libpq_ssl_params``. Prisma's ``sslcert`` is the CA and
``sslaccept=strict`` checks chain and hostname, which libpq only does in
``sslmode=verify-full``, so strict becomes ``sslrootcert`` plus
``verify-full`` whatever ``sslmode`` said (``disable`` stays off). Prisma
defaults an absent ``sslaccept`` to ``accept_invalid_certs`` and anything
else to strict. Without strict it checks nothing, so the CA is dropped and
``sslmode`` is kept as is: libpq only verifies when a root cert is present.
A URL that also carries ``sslkey`` is libpq's own client-certificate form
and is kept."""
keys: Final = frozenset(k for k, _ in pairs)
if "sslcert" not in keys or "sslkey" in keys:
return pairs
sslmode: Final = next((v for k, v in pairs if k == "sslmode"), None)
rest: Final = tuple((k, v) for k, v in pairs if k not in ("sslcert", "sslmode"))
if sslaccept in (None, "accept_invalid_certs") or sslmode == "disable":
return rest if sslmode is None else rest + (("sslmode", sslmode),)
root_cert: Final = tuple(("sslrootcert", v) for k, v in pairs if k == "sslcert" and "sslrootcert" not in keys)
return rest + root_cert + (("sslmode", "verify-full"),)
@staticmethod
def _warn_if_db_ahead_of_head(migrations_dir: str) -> None:
@ -803,7 +830,7 @@ class ProxyExtrasDBManager:
conn.execute(statement)
except psycopg.Error as e:
logger.warning(
"Could not repair invalid index %s.%s, will retry on the next startup. "
"Could not repair invalid index %s.%s, will retry on the next database setup run. "
"If this keeps happening, run `%s` by hand as the index owner. Error: %s",
index.schema,
index.name,
@ -814,16 +841,24 @@ class ProxyExtrasDBManager:
logger.info("%s invalid index %s.%s", action, index.schema, index.name)
@staticmethod
def repair_invalid_indexes(lock_timeout: str = "30s") -> bool:
def repair_invalid_indexes(
lock_timeout: str = "30s",
repair: "Callable[[psycopg.Connection[tuple[str, str, str]], _InvalidIndex], None] | None" = None,
*,
skip_request_log_indexes: bool = False,
) -> bool:
"""Rebuild LiteLLM indexes an interrupted CREATE INDEX CONCURRENTLY left
INVALID (a migration deadlock between replicas is the usual cause; the
retried migration skips them because of IF NOT EXISTS). Never raises:
returns True when no invalid index remains, False when the repair was
skipped or failed and will be retried on the next startup. Looks in the
skipped or failed and will be retried on the next database setup run. Looks in the
schema DATABASE_URL names, the only URL Prisma migrates through, but
connects over DIRECT_URL when set: the session settings, the advisory
lock and REINDEX CONCURRENTLY all need one server session, which a
transaction pooler does not give."""
transaction pooler does not give. Each rebuild holds the migration
coordinator lock on its own, like the migration job's index build, so a resolver
booting on another replica waits for one index at most. Database setup skips
managed request-log indexes so their builder owns repair without delaying serving startup."""
prisma_url: Final = os.getenv("DATABASE_URL")
if not prisma_url:
return False
@ -844,7 +879,11 @@ class ProxyExtrasDBManager:
with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
conn.execute("SET statement_timeout = 0")
conn.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(lock_timeout)))
found: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
deferred: Final = request_log_index_names(conn, schema) if skip_request_log_indexes else frozenset()
found: Final = tuple(
index for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
if index.name not in deferred
)
if not found:
return True
logger.warning(
@ -859,22 +898,95 @@ class ProxyExtrasDBManager:
if lock_row is None or not lock_row[0]:
logger.info("Another replica is already rebuilding the invalid indexes, skipping")
return False
for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema):
ProxyExtrasDBManager._repair_index(conn, index)
remaining: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
repair_one: Final = repair or ProxyExtrasDBManager._repair_index
repaired: Final = all(
ProxyExtrasDBManager._repair_under_migration_lock(conn, schema, index, repair_one)
for index in found
)
if not repaired:
return False
remaining: Final = tuple(
index for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
if index.name not in deferred
)
except psycopg.Error as e:
logger.warning("Could not check for invalid indexes, will retry on the next startup. Error: %s", e)
logger.warning(
"Could not check for invalid indexes, will retry on the next database setup run. Error: %s", e
)
return False
return not remaining
@staticmethod
def _repair_under_migration_lock(
conn: "psycopg.Connection[tuple[str, str, str]]",
schema: str,
index: _InvalidIndex,
repair: "Callable[[psycopg.Connection[tuple[str, str, str]], _InvalidIndex], None]",
) -> bool:
"""Rebuild one index under the migration coordinator lock, skipping it when a
migration job finished or dropped it in the meantime. False when another process
holds the lock, so the check waits for the next database setup run."""
with held_migration_lock(conn) as held:
if not held:
logger.info(
"Another process is building indexes under the migration lock, leaving the "
"invalid index check to the next database setup run"
)
return False
still_invalid: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
if any(found.schema == index.schema and found.name == index.name for found in still_invalid):
repair(conn, index)
return True
@staticmethod
def _setup_database_v2(use_migrate: bool) -> bool:
if not use_migrate:
return ProxyExtrasDBManager._run_database_v2(False)
from litellm_proxy_extras.migration_lock import migration_environment, migration_lock
from litellm_proxy_extras.migration_recovery import (
baseline_current_schema,
recover_completed_migration,
roll_back_failed_inert_migration,
)
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
raise RuntimeError("DATABASE_URL is required for v2 migrations")
lock_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(os.environ.get("DIRECT_URL") or database_url)
schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
def recover_completed(name: str) -> bool:
if Path(name).name != name or "\\" in name:
return False
migration: Final = Path(os.getcwd()) / "migrations" / name / "migration.sql"
if not migration.is_file():
return False
with migration_lock(lock_url) as coordinator:
return recover_completed_migration(coordinator, schema, migration) or roll_back_failed_inert_migration(
coordinator, schema, migration
)
def baseline_existing(migrations_dir: str) -> None:
with migration_lock(lock_url) as coordinator:
baseline_current_schema(
coordinator, schema, Path(migrations_dir), _get_prisma_command(), migration_environment(_get_prisma_env())
)
while not ProxyExtrasDBManager._run_database_v2(True, recover_completed, baseline_existing):
continue
return True
@staticmethod
def _run_database_v2(
use_migrate: bool,
recover_completed: Callable[[str], bool] = lambda name: False,
baseline_existing: "Callable[[str], None] | None" = None,
) -> bool:
"""
v2 migration resolver (opt-in via --use_v2_migration_resolver).
Runs `prisma migrate deploy` and handles standard recovery paths
(P3005 baseline, P3009/P3018 idempotent errors, deadlocks against a
concurrent migrate deploy). Critically, it does
Runs `prisma migrate deploy`, baselines verified existing schemas,
and recovers confirmed SQL completion or reported deadlocks. It does
NOT call `_resolve_all_migrations` — the diff-and-force recovery that
caused schema thrashing when two LiteLLM versions contended for the
same DB during rolling deploys.
@ -883,10 +995,9 @@ class ProxyExtrasDBManager:
is logged as a warning, not a fatal error — users whose DBs got into
weird shapes from the old thrashing should still be able to start.
The retry budget only counts attempts that made no progress: see
_MigrateAttemptBudget.
False requests a committed recovery checkpoint and another deploy
pass. True means every pending migration is complete.
"""
schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
if not use_migrate:
@ -919,14 +1030,22 @@ class ProxyExtrasDBManager:
original_dir = os.getcwd()
os.chdir(migrations_dir)
deploy_timeout = prisma_migrate_deploy_timeout()
budget = _MigrateAttemptBudget(attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS)
from litellm_proxy_extras.migration_lock import migration_environment, migration_lock_timeout
migration_env: Final = migration_environment(_get_prisma_env())
budget = _MigrateAttemptBudget(
attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS,
contention_seconds_left=migration_lock_timeout(),
)
try:
while not budget.exhausted:
attempt_started = time.monotonic()
try:
result = prisma_toolchain.run_prisma(
[_get_prisma_command(), "migrate", "deploy"],
timeout=deploy_timeout,
env=_get_prisma_env(),
env=migration_env,
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
return True
@ -942,8 +1061,16 @@ class ProxyExtrasDBManager:
next_budget = budget.spend()
except subprocess.CalledProcessError as e:
if "P3005" in (e.stderr or "") and baseline_existing is not None:
baseline_existing(migrations_dir)
return False
failed_migration = ProxyExtrasDBManager._v2_failed_migration_name(e.stderr or "")
if failed_migration and recover_completed(failed_migration):
return False
next_budget = ProxyExtrasDBManager._budget_after_deploy_failure(
e, budget, schema_path
e,
budget,
time.monotonic() - attempt_started,
)
if next_budget.attempts_left < budget.attempts_left:
@ -952,19 +1079,46 @@ class ProxyExtrasDBManager:
raise RuntimeError(
f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "
"attempts that made no progress (timeouts, deadlock retries, or a "
"recovery that had already run once). Check database connectivity, "
"attempts that made no progress (timeouts or deadlock retries). Check database connectivity, "
"load, and _prisma_migrations ledger state, and raise "
f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
)
finally:
os.chdir(original_dir)
@staticmethod
def _v2_failed_migration_name(stderr: str) -> "str | None":
if "P3009" in stderr:
match = re.search(r"`(\d+_[^`\r\n]+)`", stderr)
return match.group(1) if match else None
if "P3018" in stderr:
match = re.search(r"Migration name: (\d+_[^\r\n]+)", stderr)
return match.group(1) if match else None
return None
@staticmethod
def _v2_failed_migration_started_at(stderr: str, migration_name: str) -> "str | None":
match: Final = re.search(rf"`{re.escape(migration_name)}` migration started at ([^\r\n]+?) failed", stderr)
return match.group(1) if match else None
@staticmethod
def _v2_roll_back_migration_best_effort(migration_name: str) -> None:
from litellm_proxy_extras.migration_lock import migration_environment
try:
prisma_toolchain.run_prisma(
[_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
timeout=prisma_command_timeout(),
env=migration_environment(_get_prisma_env()),
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
pass
@staticmethod
def _budget_after_deploy_failure(
error: subprocess.CalledProcessError,
budget: "_MigrateAttemptBudget",
schema_path: str,
attempt_seconds: float = 0.0,
) -> "_MigrateAttemptBudget":
"""Recover from one failed `prisma migrate deploy`, and price the pass.
@ -973,37 +1127,46 @@ class ProxyExtrasDBManager:
"""
stderr = error.stderr or ""
if "P3005" in stderr and "database schema is not empty" in stderr:
logger.info("Schema exists but no migrations ledger — creating baseline")
if ProxyExtrasDBManager._create_baseline_migration(schema_path):
return budget.after_recovery("baseline")
return budget.spend()
if "P3009" in stderr:
migration_match = re.search(r"`(\d+_\S+?)`", stderr)
if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
name = migration_match.group(1)
logger.info(
f"Migration {name} failed idempotently — marking applied and retrying"
)
ProxyExtrasDBManager._mark_migration_applied(name)
return budget.after_recovery(f"resolved:{name}")
if migration_match:
migration_name = migration_match.group(1)
ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
if ledger_logs is not None and (
ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs
):
migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr)
started_at: Final = (
ProxyExtrasDBManager._v2_failed_migration_started_at(stderr, migration_name) if migration_name else None
)
if migration_name and started_at:
ledger_logs: Final = ProxyExtrasDBManager._failed_migration_logs(migration_name, started_at)
if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs:
logger.info(
"Migration %s failed in a concurrent migrate deploy "
"deadlock race, rolling its ledger row back and retrying",
migration_name,
)
ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name)
ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
return budget.spend()
if ProxyExtrasDBManager._failed_migration_recovered(migration_name, started_at):
logger.info(
"Migration %s started at %s was already rolled back or completed by a concurrent "
"migrate deploy, retrying",
migration_name,
started_at,
)
return budget.spend()
raise RuntimeError(
"Database migration failed and cannot be auto-recovered. "
f"Manual intervention required.\n\nPrisma error:\n{stderr}"
"Migration completion could not be verified. LiteLLM startup has stopped.\n\n"
f"Prisma migration history (migration name and start time):\n{stderr}\n\n"
"A migration has a start record but no successful completion record. "
"LiteLLM cannot determine whether its SQL committed from this record alone. "
"Startup stopped to avoid repeating or skipping database changes.\n\n"
"Before resolving, stop other migration runners and inspect _prisma_migrations, "
"the named migration.sql from this build, database logs, and the actual database objects and data. "
"Use the same database and this build's schema and migration files for recovery:\n"
"- Only after verifying every migration change is present, run "
"prisma migrate resolve --applied <migration_name>, then retry startup.\n"
"- Only after verifying no migration changes remain (or fully undoing partial changes), run "
"prisma migrate resolve --rolled-back <migration_name>, then retry startup. "
"This command updates history; it does not undo SQL.\n"
"Replace <migration_name> with the reported name. If the outcome remains uncertain, "
"leave migration history unchanged and contact your database administrator. "
"Repeated restarts alone will not resolve this state."
) from error
if "P3018" in stderr:
@ -1014,25 +1177,13 @@ class ProxyExtrasDBManager:
f"and retry.\n\nPrisma error:\n{stderr}"
) from error
migration_match = re.search(r"Migration name: (\d+_\S+)", stderr)
if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
name = migration_match.group(1)
migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr)
if migration_name and _MIGRATION_DEADLOCK_MARKER in stderr:
logger.info(
f"Migration {name} SQL hit idempotent error — marking applied and retrying"
)
ProxyExtrasDBManager._mark_migration_applied(name)
return budget.after_recovery(f"resolved:{name}")
if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr:
logger.info(
"Migration %s deadlocked against a concurrent "
"migrate deploy, rolling its ledger row back "
"and retrying",
migration_match.group(1),
)
ProxyExtrasDBManager._roll_back_migration_best_effort(
migration_match.group(1)
"Migration %s deadlocked against a concurrent migrate deploy, rolling its ledger row back and retrying",
migration_name,
)
ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
return budget.spend()
raise RuntimeError(
@ -1042,19 +1193,17 @@ class ProxyExtrasDBManager:
if _MIGRATION_DEADLOCK_MARKER in stderr:
logger.info(
"prisma migrate deploy attempt %s deadlocked against "
"a concurrent migrate deploy, retrying",
"prisma migrate deploy attempt %s deadlocked against a concurrent migrate deploy, retrying",
budget.attempt_number,
)
return budget.spend()
if "P1002" in stderr and "advisory lock" in stderr:
logger.info(
"prisma migrate deploy attempt %s timed out waiting for "
"the advisory lock a concurrent migrate deploy holds, retrying",
budget.attempt_number,
"Waiting for the advisory lock held by another Prisma migration; "
"contention does not spend a migration failure attempt"
)
return budget.spend()
return budget.after_contention(attempt_seconds)
raise RuntimeError(
"Database migration failed and cannot be auto-recovered. "
@ -1118,13 +1267,16 @@ class ProxyExtrasDBManager:
)
@staticmethod
def setup_database(
use_migrate: bool = False, use_v2_resolver: bool = False
) -> bool:
def setup_database(use_migrate: bool = False, use_v2_resolver: bool = False) -> bool:
"""
Set up the database using either prisma migrate or prisma db push
Uses migrations from litellm-proxy-extras package
The request-log indexes in `REQUEST_LOG_INDEXES` are not built here: the
migration job builds them through `run_migration_job`, and a serving proxy that
ran the migrations itself starts them through `start_request_log_index_build`
once it is ready to serve.
Args:
use_migrate: Whether to use prisma migrate instead of db push
use_v2_resolver: Opt into the v2 migration resolver (safer during
@ -1141,10 +1293,48 @@ class ProxyExtrasDBManager:
migrated = ProxyExtrasDBManager._run_migrations(
use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
)
if migrated:
ProxyExtrasDBManager.repair_invalid_indexes()
ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
return migrated
if not migrated:
return False
ProxyExtrasDBManager.repair_invalid_indexes(skip_request_log_indexes=True)
ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
return True
@staticmethod
def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
"""Build the indexes in `REQUEST_LOG_INDEXES` on the writer, in the schema the
migrations target. Idempotent and never raises; False when an index is still
missing or invalid, so the migration job reports it and gets rerun instead of
leaving the table unindexed until the next deploy."""
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
return True
direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
os.environ.get("DIRECT_URL") or database_url
)
schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
return build(direct_url, schema)
@staticmethod
def run_migration_job(
use_migrate: bool = False,
use_v2_resolver: bool = False,
setup: Callable[[bool, bool], bool] = setup_database,
build: Callable[[], bool] = build_request_log_indexes,
) -> bool:
"""The migration job's whole run: `setup_database`, then the request-log indexes,
built synchronously so the job exits only once they are in place. False when the
migrations failed or an index could not be built, so the Job is rerun."""
return setup(use_migrate, use_v2_resolver) and build()
@staticmethod
def start_request_log_index_build(build: Callable[[], bool] = build_request_log_indexes) -> threading.Thread:
"""A serving proxy that ran the migrations itself (schema updates not disabled)
builds the request-log indexes on a daemon thread, so a long build never delays
readiness. A build that could not finish is logged and picked up by the next boot
or the migration job."""
thread: Final = threading.Thread(target=build, name="litellm-request-log-indexes", daemon=True)
thread.start()
return thread
@staticmethod
def _run_migrations(use_migrate: bool, use_v2_resolver: bool) -> bool:
@ -1188,15 +1378,16 @@ class ProxyExtrasDBManager:
logger.info("✅ Post-migration sanity check completed")
return True
except subprocess.CalledProcessError as e:
logger.info(f"prisma db error: {e.stderr}, e: {e.stdout}")
if "P3009" in e.stderr:
stderr: Final = str(e.stderr or "")
logger.info(f"prisma db error: {stderr}, e: {e.stdout}")
if "P3009" in stderr:
# Extract the failed migration name from the error message
migration_match = re.search(
r"`(\d+_.*)` migration", e.stderr
r"`(\d+_.*)` migration", stderr
)
if migration_match:
failed_migration = migration_match.group(1)
if ProxyExtrasDBManager._is_idempotent_error(e.stderr):
if ProxyExtrasDBManager._is_idempotent_error(stderr):
logger.info(
f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied"
)
@ -1252,8 +1443,8 @@ class ProxyExtrasDBManager:
f"✅ Migration {failed_migration} marked as rolled back... retrying"
)
elif (
"P3005" in e.stderr
and "database schema is not empty" in e.stderr
"P3005" in stderr
and "database schema is not empty" in stderr
):
logger.info(
"Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system"
@ -1267,13 +1458,13 @@ class ProxyExtrasDBManager:
)
logger.info("✅ All migrations resolved.")
return True
elif "P3018" in e.stderr:
elif "P3018" in stderr:
# Check if this is a permission error or idempotent error
if ProxyExtrasDBManager._is_permission_error(e.stderr):
if ProxyExtrasDBManager._is_permission_error(stderr):
# Permission errors should NOT be marked as applied
# Extract migration name for logging
migration_match = re.search(
r"Migration name: (\d+_.*)", e.stderr
r"Migration name: (\d+_.*)", stderr
)
migration_name = (
migration_match.group(1)
@ -1283,7 +1474,7 @@ class ProxyExtrasDBManager:
logger.error(
f"❌ Migration {migration_name} failed due to insufficient permissions. "
f"Please check database user privileges. Error: {e.stderr}"
f"Please check database user privileges. Error: {stderr}"
)
# Mark as rolled back and exit with error
@ -1306,7 +1497,7 @@ class ProxyExtrasDBManager:
f"was NOT applied. Please grant necessary database permissions and retry."
) from e
elif ProxyExtrasDBManager._is_idempotent_error(e.stderr):
elif ProxyExtrasDBManager._is_idempotent_error(stderr):
# Idempotent errors mean the migration has effectively been applied
logger.info(
"Migration failed due to idempotent error (e.g., column already exists), "
@ -1314,7 +1505,7 @@ class ProxyExtrasDBManager:
)
# Extract the migration name from the error message
migration_match = re.search(
r"Migration name: (\d+_.*)", e.stderr
r"Migration name: (\d+_.*)", stderr
)
if migration_match:
migration_name = migration_match.group(1)
@ -1363,7 +1554,7 @@ class ProxyExtrasDBManager:
logger.warning(
f"P3018 error encountered but could not classify "
f"as permission or idempotent error. "
f"Error: {e.stderr}"
f"Error: {stderr}"
)
raise
else:

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.100.post1"
version = "0.4.100.post2"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.100.post1"
version = "0.4.100.post2"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

View file

@ -32,9 +32,7 @@ def _fake_migrate_deploy_failure(returncode: int, stderr: str):
def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
"""v2: a permission failure during migrate deploy raises RuntimeError."""
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
@ -50,9 +48,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
"""v2: a non-idempotent migration failure raises (no silent recovery)."""
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
@ -61,7 +57,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
'Reason: syntax error at or near "BRKN" LINE 42'
)
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -176,51 +172,33 @@ def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path))
def test_v2_resolve_specific_migration_failure_raises_runtime_error(
monkeypatch, tmp_path
):
"""If marking a migration as applied fails inside P3009 idempotent
recovery, the subprocess error must be re-raised as RuntimeError so
proxy_cli.py catches it cleanly (instead of leaking CalledProcessError)."""
def test_v2_duplicate_object_p3009_is_not_marked_applied(monkeypatch, tmp_path):
_stub_v2_env(monkeypatch, tmp_path)
monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name, started_at: "relation already exists")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
ProxyExtrasDBManager,
"_v2_roll_back_migration_best_effort",
lambda name: pytest.fail("duplicate-object errors do not prove rollback is safe"),
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
monkeypatch.setattr(
ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None
ProxyExtrasDBManager,
"_resolve_specific_migration",
lambda name: pytest.fail("duplicate-object errors do not prove all SQL completed"),
)
# First call: migrate deploy -> P3009 idempotent error.
# Recovery path tries _resolve_specific_migration; that also raises.
def _failing_resolve(*a, **kw):
raise subprocess.CalledProcessError(
returncode=1,
cmd="prisma migrate resolve --applied",
stderr="resolve failed",
output="",
)
monkeypatch.setattr(
ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve
)
stderr = (
"Error: P3009\nMigration `20260101000000_some_migration` failed\n"
"relation already exists"
)
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(
RuntimeError, match="Failed to mark migration .* as applied"
):
stderr = "Error: P3009\nMigration `20260101000000_some_migration` failed\nrelation already exists"
with patch(
"litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)
) as run:
with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
["migrate", "deploy"],
)
def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
"""v2 must never call _resolve_all_migrations — that's the bug it fixes."""
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
@ -252,9 +230,7 @@ _DEADLOCK_P3018_STDERR = (
def _stub_v2_env(monkeypatch, tmp_path):
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
monkeypatch.setattr("time.sleep", lambda _: None)
@ -272,9 +248,7 @@ def _succeed_after(failures: int, stderr: str):
return _OkResult()
calls["n"] += 1
if calls["n"] <= failures:
raise subprocess.CalledProcessError(
returncode=1, cmd=args[0], stderr=stderr, output=""
)
raise subprocess.CalledProcessError(returncode=1, cmd=args[0], stderr=stderr, output="")
return _OkResult()
return _run
@ -288,7 +262,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
rolled_back = []
monkeypatch.setattr(
ProxyExtrasDBManager,
"_roll_back_migration",
"_v2_roll_back_migration_best_effort",
lambda name: rolled_back.append(name),
)
monkeypatch.setattr(
@ -306,7 +280,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
"""v2: a deadlock on every attempt still fails after the retry budget."""
_stub_v2_env(monkeypatch, tmp_path)
monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None)
monkeypatch.setattr(ProxyExtrasDBManager, "_v2_roll_back_migration_best_effort", lambda name: None)
with patch(
"litellm_proxy_extras.prisma_toolchain.run_prisma",
@ -330,12 +304,12 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
monkeypatch.setattr(
ProxyExtrasDBManager,
"_failed_migration_logs",
lambda name: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock",
lambda name, started_at: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock",
)
rolled_back = []
monkeypatch.setattr(
ProxyExtrasDBManager,
"_roll_back_migration",
"_v2_roll_back_migration_best_effort",
lambda name: rolled_back.append(name),
)
monkeypatch.setattr(
@ -350,33 +324,28 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
assert rolled_back == ["20260415120000_health_check_latest_per_model_index"]
def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path):
"""v2: empty failed ledger logs mean a concurrent deploy moved it on."""
def test_v2_p3009_empty_ledger_logs_do_not_prove_completion(monkeypatch, tmp_path):
_stub_v2_env(monkeypatch, tmp_path)
stderr = (
"Error: P3009\n"
"migrate found failed migrations in the target database\n"
"The `20260415120000_health_check_latest_per_model_index` migration "
"started at 2026-09-01 18:46:13 UTC failed"
)
monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: "")
rolled_back = []
monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name, started_at: "")
monkeypatch.setattr(
ProxyExtrasDBManager,
"_roll_back_migration",
lambda name: rolled_back.append(name),
"_v2_roll_back_migration_best_effort",
lambda name: pytest.fail("empty logs do not prove rollback is safe"),
)
monkeypatch.setattr(
ProxyExtrasDBManager,
"_resolve_specific_migration",
lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
with patch(
"litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)
) as run:
with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
["migrate", "deploy"],
)
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
assert rolled_back == ["20260415120000_health_check_latest_per_model_index"]
def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
@ -389,15 +358,15 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
"The `20260415120000_health_check_latest_per_model_index` migration "
"started at 2026-09-01 18:46:13 UTC failed"
)
monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: None)
monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name, started_at: None)
monkeypatch.setattr(
ProxyExtrasDBManager,
"_roll_back_migration",
"_v2_roll_back_migration_best_effort",
lambda name: pytest.fail("an unreadable ledger must not trigger a retry"),
)
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -414,11 +383,11 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
monkeypatch.setattr(
ProxyExtrasDBManager,
"_failed_migration_logs",
lambda name: 'ERROR: syntax error at or near "BRKN"',
lambda name, started_at: 'ERROR: syntax error at or near "BRKN"',
)
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)

View file

@ -978,6 +978,28 @@ class PrismaManager:
os.chdir(original_dir)
return False
@staticmethod
def build_request_log_indexes() -> bool:
"""Build request-log indexes after migrations and report whether all were built."""
try:
from litellm_proxy_extras.utils import ProxyExtrasDBManager
except ImportError as e:
verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e)
return False
return ProxyExtrasDBManager.build_request_log_indexes()
@staticmethod
def start_request_log_index_build() -> None:
"""Build the request-log indexes on a daemon thread, for a serving proxy that ran the
migrations itself (`DISABLE_SCHEMA_UPDATE` unset), so a long build never delays
readiness. A build that could not finish is logged and retried on the next boot."""
try:
from litellm_proxy_extras.utils import ProxyExtrasDBManager
except ImportError as e:
verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e)
return
ProxyExtrasDBManager.start_request_log_index_build()
def should_update_prisma_schema(
disable_updates: bool | str | None = None,

View file

@ -7,9 +7,10 @@ import re
import subprocess
import sys
import urllib.parse as urlparse
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
import click
import httpx
@ -668,6 +669,33 @@ class ProxyInitializationHelpers:
return multiproc_dir
class _DatabaseSetup(Protocol):
def __call__(self, *, use_migrate: bool, use_v2_resolver: bool) -> bool: ...
@dataclass(frozen=True, slots=True)
class _DatabaseStartupDependencies:
setup: _DatabaseSetup
build: Callable[[], bool]
start: Callable[[], None]
def _setup_database_with_request_log_indexes(
skip_server_startup: bool,
*,
setup: Callable[[], bool],
build: Callable[[], bool],
start: Callable[[], None],
) -> bool:
if not setup():
return False
if skip_server_startup:
build()
return True
start()
return True
@click.command()
@click.argument("cli_args", nargs=-1)
@click.option("--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST")
@ -1360,9 +1388,21 @@ def run_server(
"diff-and-force recovery that caused the thrash).\033[0m"
)
try:
setup_ok: Final = PrismaManager.setup_database(
use_migrate=not use_prisma_db_push,
use_v2_resolver=use_v2_migration_resolver,
dependencies: Final = click.get_current_context().find_object(_DatabaseStartupDependencies) or (
_DatabaseStartupDependencies(
setup=PrismaManager.setup_database,
build=PrismaManager.build_request_log_indexes,
start=PrismaManager.start_request_log_index_build,
)
)
setup_ok: Final = _setup_database_with_request_log_indexes(
skip_server_startup,
setup=lambda: dependencies.setup(
use_migrate=not use_prisma_db_push,
use_v2_resolver=use_v2_migration_resolver,
),
build=dependencies.build,
start=dependencies.start,
)
except RuntimeError as e:
# Raised on unrecoverable migration errors: the v2

View file

@ -2,7 +2,12 @@
Runs `prisma migrate deploy` against the LiteLLM writer database using the
recovery logic in `litellm_proxy_extras.ProxyExtrasDBManager.setup_database`
(P3005 baseline + P3009/P3018 idempotent-error handling, retries, etc.).
(P3005 baseline + P3009/P3018 idempotent-error handling, retries, etc.), then
builds the request-log indexes the migrations leave out
(`litellm_proxy_extras.request_log_indexes`), waiting for them. The job exits
non-zero when an index could not be built so that it is rerun. A serving proxy
that runs the migrations itself builds the same indexes in the background once
it serves.
Env vars:
DATABASE_URL required unless it can be assembled at
@ -23,10 +28,11 @@ Env vars:
import os
import sys
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.utils import ProxyExtrasDBManager, str_to_bool
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
def main() -> int:
# Assemble DATABASE_URL from the discrete DATABASE_* env vars, matching
@ -52,7 +58,7 @@ def main() -> int:
not use_db_push,
use_v2,
)
ok = ProxyExtrasDBManager.setup_database(
ok = ProxyExtrasDBManager.run_migration_job(
use_migrate=not use_db_push,
use_v2_resolver=use_v2,
)

View file

@ -72,7 +72,7 @@ proxy = [
"mcp>=2.2.0,<3",
"httpx2>=2.5.0,<3",
"pydantic>=2.12.0,<3",
"litellm-proxy-extras==0.4.100.post1",
"litellm-proxy-extras==0.4.100.post2",
"litellm-enterprise==0.1.69.post2",
"RestrictedPython>=8.5,<9.0",
"rich>=13.9.4,<14.0",

View file

@ -7,15 +7,16 @@ anything whose cost scales with existing table size turns into downtime. A singl
plus a doubled heap that plain autovacuum will not give back.
What is banned is the row-rewriting DML behind that, not everything whose cost
scales that way. A non-concurrent `CREATE INDEX`, an `ALTER COLUMN ... TYPE` that is
not binary coercible, a volatile `DEFAULT` on a new column, a `CREATE TABLE ... AS
SELECT` or `SELECT ... INTO` filling a new table from an existing one, the rename
that pairs with one of those to swap a table out, and a `REFRESH MATERIALIZED VIEW`
all read the whole table and all pass. That is deliberate: a rule wide enough to
reach them fires on most ordinary migrations, and a marker everyone adds by reflex
stops carrying information. The outage this was written for was a backfill.
scales that way. A non-concurrent `CREATE INDEX` passes except on a request-log
table, where it blocks writes until the build finishes. An `ALTER COLUMN ... TYPE`
that is not binary coercible, a volatile `DEFAULT` on a new column, a `CREATE TABLE
... AS SELECT` or `SELECT ... INTO` filling a new table from an existing one, the
rename that pairs with one of those to swap a table out, and a `REFRESH MATERIALIZED
VIEW` all read the whole table and all pass. That is deliberate: a rule wide enough
to reach them fires on most ordinary migrations, and a marker everyone adds by
reflex stops carrying information. The outage this was written for was a backfill.
The one schema change banned outright is `ADD COLUMN ... DEFAULT` on a table in
One column change banned outright is `ADD COLUMN ... DEFAULT` on a table in
`REQUEST_LOG_TABLES`, the tables that hold a row per request. Postgres 11 stores such
a default as metadata and touches no rows, but Postgres 10, which is supported,
rewrites the whole heap and rebuilds every index under an `ACCESS EXCLUSIVE` lock,
@ -23,6 +24,13 @@ which on a spend-log-sized table is the same outage as a backfill. Every other t
is small enough that the rewrite is not worth a rule, and a column added to a log
table without a default is still free on every version.
An index on a request-log table cannot ship as a migration at all. A plain `CREATE
INDEX` blocks writes to the table until the build finishes, and `CREATE INDEX
CONCURRENTLY` is refused by Postgres on a partitioned parent, which LiteLLM_SpendLogs
is wherever the operator ran db_scripts/partition_spend_logs.sql. The migration job builds
those indexes after `migrate deploy`, concurrently and per partition, from the list in
litellm_proxy_extras/request_log_indexes.py, so that list is where a new one goes.
Flagged, per statement, by its leading keyword:
UPDATE rewrites every matching row, and `WHERE` does not bound the scan
@ -44,6 +52,8 @@ Flagged, per statement, by its leading keyword:
actions adds a column with a `DEFAULT`. An `ALTER COLUMN ... SET
DEFAULT` written after the column exists changes metadata alone, so it
passes, as does an `ADD CONSTRAINT`
CREATE only a `CREATE [UNIQUE] INDEX` on a request-log table, concurrent or
not; the migration job builds those
Referential actions (`ON DELETE CASCADE`, `ON UPDATE CASCADE`) are schema, never a
statement's leading keyword, so they pass.
@ -85,7 +95,9 @@ below line up with the statements they exempt.
Add a column and let the application populate it, or run the rewrite as an opt-in
batched job outside boot. When a rewrite is genuinely bounded and must ship inside
the migration, put `-- data-migration-ok: <reason>` on the statement or on the line
above it, naming what bounds it. The reason is required. A marker sharing a line
above it, naming what bounds it. The reason is required. A marker never exempts a
`CREATE INDEX` on a request-log table, since no bound makes that statement safe:
the migration job is the only place such an index is built. A marker sharing a line
with the statement it follows exempts that statement alone, so the next statement
down is still checked rather than picking the marker up as its own. A marker on an
`EXECUTE` or on the assignment feeding one covers the single-quoted SQL that
@ -108,6 +120,7 @@ import sys
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final
REPO_ROOT = Path(__file__).resolve().parents[2]
MIGRATIONS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
@ -115,6 +128,9 @@ MIGRATIONS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "
GRANDFATHERED = frozenset(
{
"20250425182129_add_session_id",
"20250510142544_add_session_id_index_spend_logs",
"20260228100000_add_spend_logs_composite_index",
"20250326162113_baseline",
"20260817000000_shadow_eval_multi_key",
"20260818000000_add_spend_log_timestamps",
"20260818224500_add_shadow_eval_stopped_by",
@ -139,9 +155,7 @@ WORD_OR_ASSIGN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|:=|(?<![<>!:=])=(?![=>])")
PRECEDING_WORD = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)[^A-Za-z0-9_]*$")
QUALIFIER_GAP = re.compile(r"[\s.]*")
EXPLAIN_OPTIONS = re.compile(r"\bEXPLAIN\b(?:\s+(?:ANALYZE|ANALYSE|VERBOSE)\b)+", re.IGNORECASE)
DEFINES_A_ROUTINE = re.compile(
r"\bCREATE\b(?:\s+OR\s+REPLACE)?\s+(?:FUNCTION|PROCEDURE)\b", re.IGNORECASE
)
DEFINES_A_ROUTINE = re.compile(r"\bCREATE\b(?:\s+OR\s+REPLACE)?\s+(?:FUNCTION|PROCEDURE)\b", re.IGNORECASE)
QUALIFIED_NAME = r"(?:\"[^\"]*\"|[A-Za-z_][A-Za-z0-9_$]*)"
ROUTINE_NAME = re.compile(rf"\s*(?:{QUALIFIED_NAME}\s*\.\s*)?({QUALIFIED_NAME})")
TABLE_NAME = ROUTINE_NAME
@ -204,6 +218,12 @@ statement with the bound spelled out:
-- data-migration-ok: <what bounds this>
UPDATE ...
An index on a request-log table is not a migration, and no marker exempts one. Declare it with `@@index` in
schema.prisma and add it to REQUEST_LOG_INDEXES in
litellm_proxy_extras/request_log_indexes.py under the name Prisma derives for it; the
migration job builds it after `migrate deploy`, concurrently on a plain table and per
partition on a partitioned one, which no single migration statement can do.
On Postgres 10 an `ADD COLUMN ... DEFAULT` on a request-log table rewrites the table
too. Add the column nullable with no default, then set the default in a separate
`ALTER COLUMN ... SET DEFAULT`, which never touches existing rows.
@ -215,10 +235,11 @@ class Violation:
migration: str
line: int
keyword: str
consequence: str = "rewrites existing rows at boot"
def render(self) -> str:
location = f"{MIGRATIONS_DIR.relative_to(REPO_ROOT)}/{self.migration}/migration.sql"
return f"{location}:{self.line}: {self.keyword} rewrites existing rows at boot"
return f"{location}:{self.line}: {self.keyword} {self.consequence}"
@dataclass(frozen=True, slots=True)
@ -578,6 +599,35 @@ def rewrites_a_log_table(clause: str, region: str, base: int) -> str | None:
return f"ADD COLUMN ... DEFAULT on {named.group(1)}"
def builds_a_log_index(clause: str, region: str, base: int) -> str | None:
"""The keyword to report when a `CREATE INDEX` targets a request-log table, concurrent or
not: a plain build blocks writes for its whole duration, and a concurrent one fails with
P3018 on a partitioned parent, so the migration job builds those instead."""
created: Final[re.Match[str] | None] = re.match(
r"\s*CREATE\s+(?:UNIQUE\s+)?INDEX\b(?:\s+CONCURRENTLY\b)?", clause, re.IGNORECASE
)
if created is None:
return None
on: Final[re.Match[str] | None] = re.search(r"\bON\b(?:\s+ONLY\b)?", clause[created.end() :], re.IGNORECASE)
if on is None:
return None
named: Final[re.Match[str] | None] = TABLE_NAME.match(
region, skip_comments(region, base + created.end() + on.end())
)
if named is None or named.group(1).strip('"') not in REQUEST_LOG_TABLES:
return None
return f"CREATE INDEX on {named.group(1)}"
def consequence_of(found: str) -> str:
if found.startswith("CREATE INDEX"):
return (
"blocks writes until the build finishes, or fails on a partitioned table; "
"add it to REQUEST_LOG_INDEXES in litellm_proxy_extras/request_log_indexes.py instead"
)
return "rewrites existing rows at boot"
def skip_comments(sql: str, start: int) -> int:
index = start
while index < len(sql):
@ -746,8 +796,7 @@ def read_markers(sql: str) -> Markers:
return Markers(
sql,
tuple(
Marker(match.start(), match.end(), alone_on_its_line(sql, match.start()))
for match in MARKER.finditer(sql)
Marker(match.start(), match.end(), alone_on_its_line(sql, match.start())) for match in MARKER.finditer(sql)
),
)
@ -760,9 +809,7 @@ def scan(sql: str, migration: str, markers: Markers) -> Iterator[Violation]:
yield from scan_region(sql, sql, migration, markers, 0)
def scan_region(
document: str, region: str, migration: str, markers: Markers, offset: int
) -> Iterator[Violation]:
def scan_region(document: str, region: str, migration: str, markers: Markers, offset: int) -> Iterator[Violation]:
"""Violations in one region of `document`, whose text begins at `offset`. Positions are
always counted against the whole document, so a statement nested in a dollar-quoted body
reports its real file line and lines up with the markers read from that file. A single-quoted
@ -790,13 +837,18 @@ def scan_region(
offset + start,
)
keyword = offending_keyword(clause)
if exempt:
continue
found = keyword or rewrites_a_log_table(clause, region, base)
index = builds_a_log_index(clause, region, base)
found = (
index if exempt else offending_keyword(clause) or rewrites_a_log_table(clause, region, base) or index
)
if found is None:
continue
yield Violation(migration, line_of(document, offset + keyword_start(clause, base)), found)
yield Violation(
migration,
line_of(document, offset + keyword_start(clause, base)),
found,
consequence_of(found),
)
for body in bodies:
if not runs_when_applied(masked, region, bodies, runnable, identifiers, body):

View file

@ -16,6 +16,10 @@ import psutil
from integration._support.client import Gateway
DB_PUSH: Final = ("--use_prisma_db_push",)
MIGRATE_DEPLOY: Final = ("--use_v2_migration_resolver",)
LEGACY_MIGRATE_DEPLOY: Final = ()
def in_group(process: psutil.Process, group: int) -> bool:
try:
@ -62,9 +66,16 @@ def owned_proxy(
config: Path | None = None,
remove_environment: tuple[str, ...] = (),
workers: int = 1,
database_setup: tuple[str, ...] = DB_PUSH,
) -> Iterator[Gateway]:
with owned_proxy_process(
gateway, directory, overrides, config=config, remove_environment=remove_environment, workers=workers
gateway,
directory,
overrides,
config=config,
remove_environment=remove_environment,
workers=workers,
database_setup=database_setup,
) as owned:
yield owned.gateway
@ -78,6 +89,7 @@ def owned_proxy_process(
config: Path | None = None,
remove_environment: tuple[str, ...] = (),
workers: int = 1,
database_setup: tuple[str, ...] = DB_PUSH,
) -> Iterator[OwnedProxy]:
with socket.socket() as reserve:
reserve.bind(("127.0.0.1", 0))
@ -109,7 +121,7 @@ def owned_proxy_process(
str(workers),
"--telemetry",
"False",
"--use_prisma_db_push",
*database_setup,
"--enforce_prisma_migration_check",
],
cwd=root,

View file

@ -0,0 +1,363 @@
import os
import shutil
import subprocess
import sys
from collections.abc import Mapping
from dataclasses import dataclass
from itertools import product
from pathlib import Path
from types import MappingProxyType
from typing import Final
import psycopg
import pytest
from integration._support.client import Gateway, eventually
from integration._support.database import scratch_database
from integration._support.process import LEGACY_MIGRATE_DEPLOY, MIGRATE_DEPLOY, owned_proxy_process
from psycopg import sql
from psycopg.rows import class_row
REPO_ROOT: Final = Path(__file__).resolve().parents[3]
PRISMA_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras"
PARTITION_SCRIPT: Final = REPO_ROOT / "db_scripts" / "partition_spend_logs.sql"
SHIPPED_MIGRATIONS: Final = tuple(sorted(path.name for path in (PRISMA_DIR / "migrations").iterdir() if path.is_dir()))
API_KEY_INDEX_MIGRATION: Final = "20260823000000_add_spend_logs_api_key_starttime_index"
CALL_ID_INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index"
ORIGINAL_MIGRATION_SQL: Final = MappingProxyType(
{
API_KEY_INDEX_MIGRATION: (
"-- CreateIndex\n"
'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" '
'ON "LiteLLM_SpendLogs"("api_key", "startTime");\n'
),
CALL_ID_INDEX_MIGRATION: (
"-- CreateIndex\n"
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" '
'ON "LiteLLM_SpendLogs"("litellm_call_id");\n'
),
}
)
MIGRATION_JOB_SECONDS: Final = 300
INDEXES_IN_PLACE: Final = "Request-log indexes are all in place"
INDEX_BUILD_LINES: Final = ("Building index", "Attached index")
BUILD_SECONDS: Final = 60
SPEND_LOGS_INDEXES: Final = ("LiteLLM_SpendLogs_api_key_startTime_idx", "LiteLLM_SpendLogs_litellm_call_id_idx")
POPULATED_PARTITIONS: Final = MappingProxyType(
{
"LiteLLM_SpendLogs_p2026_08": ("2026-08-01", "2026-09-01"),
"LiteLLM_SpendLogs_p2026_09": ("2026-09-01", "2026-10-01"),
}
)
DEFAULT_PARTITION: Final = "LiteLLM_SpendLogs_pdefault"
ROWS_PER_PARTITION: Final = 500
PARTITIONED_PARENT_ERROR: Final = 'cannot create index on partitioned table "LiteLLM_SpendLogs" concurrently'
@dataclass(frozen=True, slots=True)
class Resolver:
"""One migration resolver as the serving proxy selects it (CLI flags) and as the
migration job selects it (environment)."""
proxy_flags: tuple[str, ...]
job_environment: Mapping[str, str]
V2: Final = Resolver(MIGRATE_DEPLOY, MappingProxyType({"USE_V2_MIGRATION_RESOLVER": "true"}))
LEGACY: Final = Resolver(LEGACY_MIGRATE_DEPLOY, MappingProxyType({"USE_V2_MIGRATION_RESOLVER": "false"}))
RESOLVERS: Final = pytest.mark.parametrize("resolver", (V2, LEGACY), ids=("v2", "legacy"))
def release_layout(directory: Path, migrations: tuple[str, ...]) -> Path:
"""The Prisma layout of the release that shipped `migrations`: the two index migrations
carry the SQL they shipped with, not the inert files of this build."""
(directory / "migrations").mkdir(parents=True)
shutil.copy(PRISMA_DIR / "schema.prisma", directory / "schema.prisma")
shutil.copy(PRISMA_DIR / "migrations" / "migration_lock.toml", directory / "migrations" / "migration_lock.toml")
for name in migrations:
shutil.copytree(PRISMA_DIR / "migrations" / name, directory / "migrations" / name)
for name, original in ORIGINAL_MIGRATION_SQL.items():
if name in migrations:
(directory / "migrations" / name / "migration.sql").write_text(original)
return directory / "schema.prisma"
def migrate_deploy(database_url: str, schema: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(schema)],
capture_output=True,
text=True,
timeout=300,
env={**os.environ, "DATABASE_URL": database_url},
)
def migration_job(database_url: str, resolver: Resolver) -> subprocess.CompletedProcess[str]:
"""The migrations image entrypoint, as the helm migration Job runs it."""
return subprocess.run(
[sys.executable, "-I", str(REPO_ROOT / "migrations" / "run.py")],
capture_output=True,
text=True,
timeout=MIGRATION_JOB_SECONDS,
cwd=REPO_ROOT,
env={**os.environ, "DATABASE_URL": database_url, **resolver.job_environment},
)
def migration_cli(database_url: str, gateway: Gateway, resolver: Resolver) -> subprocess.CompletedProcess[str]:
"""The proxy CLI migrates, builds the indexes and exits by the migrations result."""
return subprocess.run(
[
sys.executable,
"-P",
"-m",
"integration._support.proxy",
"--config",
"tests/integration/proxy_config.yaml",
*resolver.proxy_flags,
"--skip_server_startup",
"--enforce_prisma_migration_check",
],
capture_output=True,
text=True,
timeout=MIGRATION_JOB_SECONDS,
cwd=REPO_ROOT,
env={**os.environ, "DATABASE_URL": database_url, "LITELLM_MASTER_KEY": gateway.key},
)
def deploy_schema_before_the_index_migrations(database_url: str, directory: Path) -> None:
older: Final = tuple(name for name in SHIPPED_MIGRATIONS if name < API_KEY_INDEX_MIGRATION)
deployed: Final = migrate_deploy(database_url, release_layout(directory / "older-release", older))
assert deployed.returncode == 0, deployed.stdout + deployed.stderr
def deploy_the_original_index_migrations(database_url: str, directory: Path) -> subprocess.CompletedProcess[str]:
"""Boot the v1.103.0 layout once: on a partitioned table its CONCURRENTLY call_id index
fails with P3018 and leaves the ledger row unfinished, on a plain table both apply."""
return migrate_deploy(database_url, release_layout(directory / "v1.103.0", SHIPPED_MIGRATIONS))
def fail_the_call_id_index_migration_like_the_shipped_release(database_url: str, directory: Path) -> None:
deployed: Final = deploy_the_original_index_migrations(database_url, directory)
assert deployed.returncode != 0, deployed.stdout
assert "P3018" in deployed.stderr and PARTITIONED_PARENT_ERROR in deployed.stderr, deployed.stderr
assert ledger(database_url)[CALL_ID_INDEX_MIGRATION] is False
def partition_spend_logs(database_url: str) -> None:
with psycopg.connect(database_url, autocommit=True) as connection:
connection.execute(PARTITION_SCRIPT.read_bytes())
for partition, (start, stop) in POPULATED_PARTITIONS.items():
add_partition(connection, partition, start, stop)
connection.execute(
'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "api_key", "startTime", "endTime") '
"SELECT %s || n, 'acompletion', 'sk-' || (n %% 7), %s::timestamp + (n * interval '1 minute'), "
"%s::timestamp + (n * interval '1 minute') + interval '1 second' FROM generate_series(1, %s) AS n",
(partition, start, start, ROWS_PER_PARTITION),
)
connection.execute(
'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "startTime", "endTime") '
"SELECT 'default-' || n, 'acompletion', '2026-07-01'::timestamp + (n * interval '1 minute'), "
"'2026-07-01'::timestamp + (n * interval '1 minute') FROM generate_series(1, %s) AS n",
(ROWS_PER_PARTITION,),
)
@dataclass(frozen=True, slots=True)
class _LedgerRow:
name: str
finished: bool
@dataclass(frozen=True, slots=True)
class _IndexRow:
index: str
valid: bool
@dataclass(frozen=True, slots=True)
class _OidRow:
index: str
oid: int
@dataclass(frozen=True, slots=True)
class _AttachedRow:
partition: str
parent_index: str
valid: bool
def ledger(database_url: str) -> Mapping[str, bool]:
"""Every migration in the ledger that was not rolled back, mapped to whether it finished."""
with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_LedgerRow)) as cursor:
rows: Final = cursor.execute(
'SELECT migration_name AS name, finished_at IS NOT NULL AS finished FROM "_prisma_migrations" '
"WHERE rolled_back_at IS NULL ORDER BY migration_name"
).fetchall()
return MappingProxyType({row.name: row.finished for row in rows})
def parent_index_validity(database_url: str) -> Mapping[str, bool]:
with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_IndexRow)) as cursor:
rows: Final = cursor.execute(
"SELECT c.relname AS index, x.indisvalid AS valid FROM pg_index x JOIN pg_class c ON c.oid = x.indexrelid "
"WHERE x.indrelid = '\"LiteLLM_SpendLogs\"'::regclass AND c.relname = ANY(%s)",
(list(SPEND_LOGS_INDEXES),),
).fetchall()
return MappingProxyType({row.index: row.valid for row in rows})
def index_oids(database_url: str) -> Mapping[str, int]:
"""index name -> oid for the SpendLogs indexes on the parent or any partition; a rebuild changes the oid."""
with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_OidRow)) as cursor:
rows: Final = cursor.execute(
"SELECT c.relname AS index, c.oid::int AS oid FROM pg_index x JOIN pg_class c ON c.oid = x.indexrelid "
"WHERE x.indrelid = '\"LiteLLM_SpendLogs\"'::regclass OR x.indrelid IN "
"(SELECT inhrelid FROM pg_inherits WHERE inhparent = '\"LiteLLM_SpendLogs\"'::regclass)"
).fetchall()
return MappingProxyType({row.index: row.oid for row in rows})
def attached_partition_indexes(database_url: str) -> frozenset[tuple[str, str, bool]]:
"""(partition, parent index, child is valid) for every child index attached under a SpendLogs parent index."""
with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_AttachedRow)) as cursor:
rows: Final = cursor.execute(
"SELECT part.relname AS partition, parent_index.relname AS parent_index, child.indisvalid AS valid "
"FROM pg_inherits attached "
"JOIN pg_class parent_index ON parent_index.oid = attached.inhparent "
"JOIN pg_index child ON child.indexrelid = attached.inhrelid "
"JOIN pg_class part ON part.oid = child.indrelid "
"WHERE parent_index.relname = ANY(%s)",
(list(SPEND_LOGS_INDEXES),),
).fetchall()
return frozenset((row.partition, row.parent_index, row.valid) for row in rows)
def expected_attachments(partitions: tuple[str, ...]) -> frozenset[tuple[str, str, bool]]:
return frozenset((partition, index, True) for partition, index in product(partitions, SPEND_LOGS_INDEXES))
def add_partition(connection: psycopg.Connection[tuple[object, ...]], partition: str, start: str, stop: str) -> None:
connection.execute(
sql.SQL('CREATE TABLE {} PARTITION OF "LiteLLM_SpendLogs" FOR VALUES FROM ({}) TO ({})').format(
sql.Identifier(partition), sql.Literal(start), sql.Literal(stop)
)
)
def assert_ready(booted_gateway: Gateway) -> None:
readiness: Final = booted_gateway.request("GET", "/health/readiness")
assert readiness.status_code == 200, readiness.text
assert readiness.json()["db"] == "connected", readiness.text
def assert_both_indexes_cover_every_partition(database_url: str) -> None:
"""Every populated partition's index is attached and valid, both parents are valid, and
a partition created afterwards inherits both indexes."""
populated: Final = (*POPULATED_PARTITIONS, DEFAULT_PARTITION)
assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS}
assert attached_partition_indexes(database_url) == expected_attachments(populated)
assert parent_index_validity(database_url) == {index: True for index in SPEND_LOGS_INDEXES}
with psycopg.connect(database_url, autocommit=True) as connection:
add_partition(connection, "LiteLLM_SpendLogs_p2026_10", "2026-10-01", "2026-11-01")
assert attached_partition_indexes(database_url) == expected_attachments((*populated, "LiteLLM_SpendLogs_p2026_10"))
def assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(
gateway: Gateway, directory: Path, database_url: str, resolver: Resolver
) -> None:
"""The serving proxy applies the inert files, reports ready, and its background build
finds every index already there, so it builds nothing and the catalog is untouched."""
oids: Final = index_oids(database_url)
with owned_proxy_process(
gateway, directory, {"DATABASE_URL": database_url}, database_setup=resolver.proxy_flags
) as booted:
assert_ready(booted.gateway)
log: Final = eventually(
lambda: booted.log.read_text(errors="replace"), lambda text: INDEXES_IN_PLACE in text, seconds=BUILD_SECONDS
)
assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS}
assert not any(line in log for line in INDEX_BUILD_LINES), log[-4000:]
assert index_oids(database_url) == oids
@RESOLVERS
def test_the_migration_job_gives_a_partitioned_table_at_the_pre_index_schema_both_indexes_per_partition(
gateway: Gateway, tmp_path: Path, resolver: Resolver
) -> None:
with scratch_database() as database_url:
deploy_schema_before_the_index_migrations(database_url, tmp_path)
partition_spend_logs(database_url)
job: Final = migration_job(database_url, resolver)
assert job.returncode == 0, job.stdout + job.stderr
assert INDEXES_IN_PLACE in job.stderr + job.stdout, job.stdout + job.stderr
assert_both_indexes_cover_every_partition(database_url)
assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(gateway, tmp_path, database_url, resolver)
@RESOLVERS
def test_the_migration_job_heals_a_partitioned_table_left_with_the_failed_call_id_ledger_row(
gateway: Gateway, tmp_path: Path, resolver: Resolver
) -> None:
with scratch_database() as database_url:
deploy_schema_before_the_index_migrations(database_url, tmp_path)
partition_spend_logs(database_url)
fail_the_call_id_index_migration_like_the_shipped_release(database_url, tmp_path)
job: Final = migration_job(database_url, resolver)
assert job.returncode == 0, job.stdout + job.stderr
assert_both_indexes_cover_every_partition(database_url)
assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(gateway, tmp_path, database_url, resolver)
@RESOLVERS
def test_the_migration_job_leaves_a_plain_table_that_applied_the_original_index_migrations_alone(
gateway: Gateway, tmp_path: Path, resolver: Resolver
) -> None:
with scratch_database() as database_url:
deploy_schema_before_the_index_migrations(database_url, tmp_path)
deployed: Final = deploy_the_original_index_migrations(database_url, tmp_path)
assert deployed.returncode == 0, deployed.stdout + deployed.stderr
before: Final = index_oids(database_url)
assert set(SPEND_LOGS_INDEXES) <= set(before), before
job: Final = migration_job(database_url, resolver)
assert job.returncode == 0, job.stdout + job.stderr
assert INDEXES_IN_PLACE in job.stderr + job.stdout, job.stdout + job.stderr
assert "Building index" not in job.stderr + job.stdout, job.stdout + job.stderr
assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS}
assert index_oids(database_url) == before
assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(gateway, tmp_path, database_url, resolver)
@RESOLVERS
def test_a_serving_proxy_that_runs_the_migrations_itself_builds_both_indexes_after_it_is_ready(
gateway: Gateway, tmp_path: Path, resolver: Resolver
) -> None:
"""A deployment that runs migrate deploy from the serving proxy and never runs the
migration job answers readiness with the inert files applied, then its background build
puts both indexes on every partition."""
with scratch_database() as database_url:
deploy_schema_before_the_index_migrations(database_url, tmp_path)
partition_spend_logs(database_url)
assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in parent_index_validity(database_url)
with owned_proxy_process(
gateway, tmp_path, {"DATABASE_URL": database_url}, database_setup=resolver.proxy_flags
) as booted:
assert_ready(booted.gateway)
assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS}
log: Final = eventually(
lambda: booted.log.read_text(errors="replace"),
lambda text: INDEXES_IN_PLACE in text,
seconds=BUILD_SECONDS,
)
assert "Building index" in log and "Attached index" in log, log[-4000:]
assert_both_indexes_cover_every_partition(database_url)
def test_the_cli_run_as_a_migration_job_builds_both_indexes_before_it_exits(gateway: Gateway, tmp_path: Path) -> None:
with scratch_database() as database_url:
deploy_schema_before_the_index_migrations(database_url, tmp_path)
partition_spend_logs(database_url)
job: Final = migration_cli(database_url, gateway, V2)
assert job.returncode == 0, job.stdout + job.stderr
assert_both_indexes_cover_every_partition(database_url)

View file

@ -2,7 +2,12 @@ import glob
import os
import re
import sys
from collections.abc import Callable
from dataclasses import dataclass
import threading
from pathlib import Path
from queue import SimpleQueue
from typing import Final
import pytest
@ -13,10 +18,10 @@ sys.path.insert(
),
)
from litellm_proxy_extras.request_log_indexes import filter_request_log_index_diff
from litellm_proxy_extras.utils import (
PARTITIONED_SPEND_LOGS_PUSH_ERROR,
ProxyExtrasDBManager,
filter_hand_built_spend_logs_index_diff,
filter_partitioned_spend_logs_diff,
)
@ -29,6 +34,122 @@ _MIGRATIONS_DIR = os.path.abspath(
)
def test_coordination_failure_does_not_misclassify_fatal_password_as_unreachable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import psycopg
from litellm_proxy_extras.migration_lock import _coordination_failure
database_url: Final = "postgresql://u:FATAL@127.0.0.1:5432/db"
monkeypatch.setenv("DATABASE_URL", database_url)
error: Final = psycopg.OperationalError(
'connection failed: connection to server at "127.0.0.1", port 5432 failed: '
'FATAL: password authentication failed for user "u"'
)
result: Final = _coordination_failure(database_url, error, connecting=True)
assert result.startswith("Lost or could not establish v2 migration coordination with the database: ")
assert "P1001" not in result
@pytest.mark.parametrize(
("message", "detail"),
[
(
'connection failed: connection to server at "127.0.0.1", port 5432 failed: SSL error: '
"certificate verify failed",
"certificate verify failed",
),
(
'connection failed: connection to server at "127.0.0.1", port 5432 failed: server certificate '
'for "db.internal" does not match host name "127.0.0.1"',
"does not match host name",
),
(
'connection failed: connection to server at "127.0.0.1", port 5432 failed: server does not support '
"SSL, but SSL was required",
"SSL was required",
),
],
)
def test_coordination_failure_does_not_label_tls_failures_as_unreachable(
monkeypatch: pytest.MonkeyPatch,
message: str,
detail: str,
) -> None:
import psycopg
from litellm_proxy_extras.migration_lock import _coordination_failure
database_url: Final = "postgresql://u:pw@127.0.0.1:5432/db"
monkeypatch.setenv("DATABASE_URL", database_url)
error: Final = psycopg.OperationalError(message)
result: Final = _coordination_failure(database_url, error, connecting=True)
assert result.startswith("Lost or could not establish v2 migration coordination with the database: ")
assert "P1001" not in result
assert detail in result
@pytest.mark.parametrize(
("database_url", "message", "host"),
[
(
"postgresql://u:pw@certificate-db.internal:5432/db",
'connection failed: connection to server at "certificate-db.internal" (10.0.0.5), port 5432 failed: '
"Connection refused",
"certificate-db.internal",
),
(
"postgresql://u:pw@SSL-primary:5432/db",
'connection failed: connection to server at "SSL-primary" (10.0.0.6), port 5432 failed: '
"Connection refused",
"ssl-primary",
),
(
"postgresql://u:pw@certificate-db.invalid:5432/db",
"connection failed: failed to resolve host 'certificate-db.invalid': [Errno -2] "
"Name or service not known",
"certificate-db.invalid",
),
],
)
def test_coordination_failure_labels_unreachable_hosts_named_like_tls_markers(
monkeypatch: pytest.MonkeyPatch,
database_url: str,
message: str,
host: str,
) -> None:
import psycopg
from litellm_proxy_extras.migration_lock import _coordination_failure
monkeypatch.setenv("DATABASE_URL", database_url)
error: Final = psycopg.OperationalError(message)
result: Final = _coordination_failure(database_url, error, connecting=True)
assert f"P1001: Can't reach database server at `{host}:5432`" in result
def test_coordination_failure_labels_connection_refused_as_unreachable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import psycopg
from litellm_proxy_extras.migration_lock import _coordination_failure
database_url: Final = "postgresql://u:pw@127.0.0.1:5432/db"
monkeypatch.setenv("DATABASE_URL", database_url)
error: Final = psycopg.OperationalError(
'connection failed: connection to server at "127.0.0.1", port 5432 failed: Connection refused'
)
result: Final = _coordination_failure(database_url, error, connecting=True)
assert result.startswith("Lost or could not establish v2 migration coordination with the database: ")
assert "P1001: Can't reach database server at `127.0.0.1:5432`" in result
def test_custom_prisma_dir(monkeypatch):
import tempfile
@ -644,14 +765,14 @@ class TestHandBuiltSpendLogsIndexDriftFilter:
],
)
def test_hand_built_index_creation_is_removed(self, statement):
assert filter_hand_built_spend_logs_index_diff(f"-- CreateIndex\n{statement};\n") == ""
assert filter_request_log_index_diff(f"-- CreateIndex\n{statement};\n") == ""
def test_other_spend_logs_indexes_and_statements_are_kept_verbatim(self):
kept = (
'CREATE INDEX "LiteLLM_SpendLogs_session_id_idx" ON "LiteLLM_SpendLogs"("session_id");\n'
'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;\n'
)
assert filter_hand_built_spend_logs_index_diff(kept) == kept
assert filter_request_log_index_diff(kept) == kept
class TestPartitionedSpendLogsPushGuard:
@ -761,6 +882,9 @@ class TestSpendLogsPartitionDetectionMissingPsycopg:
_ATTEMPT_BUDGET = 4
_P3009_MIGRATION_NAME = "20260415120000_health_check_latest_per_model_index"
_P3009_STARTED_AT = "2026-10-02 23:20:56.439594 UTC"
_P3009_DEADLOCK_LOGS = "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock on transaction 991"
_P3005_STDERR = """Error: P3005
@ -782,12 +906,120 @@ ERROR: relation "SomeTable" already exists
"""
def _p3009_stderr(migration_name: str, started_at: str) -> str:
return (
"Error: P3009\n\n"
"migrate found failed migrations in the target database, new migrations will not be applied. "
"Read more about how to resolve migration issues in a production database: "
"https://pris.ly/d/migrate-resolve\n"
f"The `{migration_name}` migration started at {started_at} failed\n"
)
@dataclass(frozen=True, slots=True)
class _LedgerRow:
migration_name: str
started_at: str
finished: bool = False
rolled_back: bool = False
logs: str | None = None
@dataclass(frozen=True, slots=True)
class _LedgerCursor:
row: tuple[object, ...] | None = None
def fetchone(self) -> tuple[object, ...] | None:
return self.row
def fetchall(self) -> tuple[tuple[object, ...], ...]:
return ()
class _LedgerConnection:
def __init__(self, ledger: "_FakeLedger") -> None:
self.ledger = ledger
def __enter__(self) -> "_LedgerConnection":
return self
def __exit__(self, *args: object) -> None:
return None
def execute(self, query: object, params: tuple[object, ...] = ()) -> _LedgerCursor:
return self.ledger.execute(query, params)
class _FakeLedger:
def __init__(self, at_error: tuple[_LedgerRow, ...], after_peer: tuple[_LedgerRow, ...]) -> None:
self.rows = at_error
self.after_peer = after_peer
self._peer_observed = False
def connect(self, *args: object, **kwargs: object) -> _LedgerConnection:
return _LedgerConnection(self)
def execute(self, query: object, params: tuple[object, ...]) -> _LedgerCursor:
text: Final = str(query)
if "WHERE migration_name = %s" not in text or not params:
return _LedgerCursor()
if not self._peer_observed:
self.rows = self.after_peer
self._peer_observed = True
matching: Final = tuple(
row
for row in self.rows
if row.migration_name == params[0] and (len(params) == 1 or row.started_at == params[1])
)
if "rolled_back_at IS NULL" in text:
unresolved: Final = next((row for row in matching if not row.finished and not row.rolled_back), None)
return _LedgerCursor((unresolved.logs,) if unresolved else None)
if "IS NOT NULL" in text:
resolved: Final = next((row for row in matching if row.finished or row.rolled_back), None)
return _LedgerCursor((1,) if resolved else None)
return _LedgerCursor()
@pytest.mark.parametrize(
"pooled,direct,expected",
(
("postgresql://pool/db?pgbouncer=true", None, "postgresql://pool/db?pgbouncer=true"),
("postgresql://pool/db?pgbouncer=true", "postgresql://writer/db", "postgresql://writer/db?schema=public"),
(
"postgresql://pool/db?schema=tenant%20one&pgbouncer=true",
"postgresql://writer/db?sslmode=require&schema=wrong",
"postgresql://writer/db?sslmode=require&schema=tenant+one",
),
),
)
def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema(
pooled: str, direct: str | None, expected: str
) -> None:
from litellm_proxy_extras.migration_lock import migration_environment
environment: Final = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"}
configured: Final = {**environment, **({"DIRECT_URL": direct} if direct else {})}
migrated: Final = migration_environment(configured)
assert migrated["DATABASE_URL"] == expected
assert migrated["PRISMA_OFFLINE_MODE"] == "true"
assert configured["DATABASE_URL"] == pooled
class _MigrateDeployHarness:
"""Drives _setup_database_v2 with a scripted sequence of
`prisma migrate deploy` outcomes, with every recovery command faked out so
nothing touches a database or the packaged migrations directory."""
def __init__(self, monkeypatch, tmp_path, outcomes, repeat_last=False):
def __init__(
self,
monkeypatch,
tmp_path,
outcomes,
repeat_last=False,
confirmed_migrations=(),
ledger: "_FakeLedger | None" = None,
):
import subprocess as subprocess_module
import litellm_proxy_extras.utils as utils_module
@ -798,16 +1030,9 @@ class _MigrateDeployHarness:
self._outcomes = list(outcomes)
self._repeat_last = repeat_last
self._subprocess_module = subprocess_module
self.confirmed_migrations = set(confirmed_migrations)
monkeypatch.delenv("DATABASE_URL", raising=False)
monkeypatch.setattr(
ProxyExtrasDBManager, "_get_prisma_dir", staticmethod(lambda: str(tmp_path))
)
monkeypatch.setattr(
ProxyExtrasDBManager,
"_create_baseline_migration",
staticmethod(self._fake_baseline),
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", staticmethod(lambda: str(tmp_path)))
monkeypatch.setattr(
ProxyExtrasDBManager,
"_roll_back_migration",
@ -818,14 +1043,22 @@ class _MigrateDeployHarness:
"_resolve_specific_migration",
staticmethod(self.resolved.append),
)
if ledger is None:
monkeypatch.delenv("DATABASE_URL", raising=False)
else:
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr("psycopg.connect", ledger.connect)
monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", self._fake_run)
monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: {})
monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None)
self.baseline_succeeds = True
def _fake_baseline(self, *args, **kwargs):
self.baselines += 1
return self.baseline_succeeds
if not self.baseline_succeeds:
raise RuntimeError("The existing schema was not verified")
def _next_outcome(self):
if self._outcomes:
@ -845,79 +1078,214 @@ class _MigrateDeployHarness:
raise self._subprocess_module.CalledProcessError(1, cmd, stderr=outcome)
def run(self):
return ProxyExtrasDBManager._setup_database_v2(use_migrate=True)
while not ProxyExtrasDBManager._run_database_v2(
use_migrate=True,
recover_completed=self._fake_recovery,
baseline_existing=self._fake_baseline,
):
continue
return True
def _fake_recovery(self, name: str) -> bool:
if name not in self.confirmed_migrations:
return False
self.confirmed_migrations.remove(name)
self.resolved.append(name)
return True
class TestConcurrentP3009Recovery:
@pytest.mark.parametrize(
"after_peer",
(
(_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS),),
(
_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS),
_LedgerRow(_P3009_MIGRATION_NAME, "2026-10-02 23:21:11.539224 UTC"),
),
(_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, finished=True, logs=_P3009_DEADLOCK_LOGS),),
),
ids=("rolled-back", "rolled-back-beside-a-fresh-in-flight-row", "finished"),
)
def test_a_p3009_row_a_peer_already_recovered_is_retried(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
after_peer: tuple[_LedgerRow, ...],
) -> None:
deadlocked_row: Final = _LedgerRow(
_P3009_MIGRATION_NAME,
_P3009_STARTED_AT,
logs=_P3009_DEADLOCK_LOGS,
)
harness: Final = _MigrateDeployHarness(
monkeypatch,
tmp_path,
[_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT), "ok"],
ledger=_FakeLedger(at_error=(deadlocked_row,), after_peer=after_peer),
)
assert harness.run() is True
assert len(harness.deploy_calls) == 2
@pytest.mark.parametrize(
"ledger_rows",
(
(
_LedgerRow(
_P3009_MIGRATION_NAME,
_P3009_STARTED_AT,
logs='ERROR: syntax error at or near "SLECT"',
),
),
(
_LedgerRow(
_P3009_MIGRATION_NAME,
_P3009_STARTED_AT,
logs='ERROR: syntax error at or near "SLECT"',
),
_LedgerRow(
_P3009_MIGRATION_NAME,
"2026-10-02 23:19:40.120000 UTC",
rolled_back=True,
logs=_P3009_DEADLOCK_LOGS,
),
),
),
ids=("only-row", "beside-a-recovered-earlier-attempt"),
)
def test_an_unresolved_p3009_row_without_the_deadlock_marker_stops(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
ledger_rows: tuple[_LedgerRow, ...],
) -> None:
harness: Final = _MigrateDeployHarness(
monkeypatch,
tmp_path,
[_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT)],
ledger=_FakeLedger(at_error=ledger_rows, after_peer=ledger_rows),
)
with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
harness.run()
assert len(harness.deploy_calls) == 1
class TestMigrateDeployAttemptAccounting:
"""A `prisma db push` database has a full schema and no ledger, so the v2
resolver baselines it and then works through every migration whose objects
already exist. Those recoveries make progress, so they must not spend the
retry budget, which is there to stop a run that is getting nowhere."""
def test_a_push_created_database_finishes_bootstrapping(
self, monkeypatch, tmp_path
):
already_there = [
"20250329084805_new_cron_job_table",
"20250806095134_rename_alias_to_server_name_mcp_table",
"20260224203854_add_agent_object_permissions_table",
"20260301120000_fourth_table",
"20260302120000_fifth_table",
"20260303120000_sixth_table",
]
def test_a_push_created_database_finishes_bootstrapping(self, monkeypatch, tmp_path):
harness = _MigrateDeployHarness(
monkeypatch,
tmp_path,
[_P3005_STDERR]
+ [_p3018_stderr(name) for name in already_there]
+ ["ok"],
[_P3005_STDERR, "ok"],
)
assert harness.run() is True
assert harness.baselines == 1
assert harness.resolved == already_there
assert len(harness.deploy_calls) == len(already_there) + 2
assert harness.resolved == []
assert len(harness.deploy_calls) == 2
def test_repeated_recovery_of_one_migration_still_gives_up(
self, monkeypatch, tmp_path
):
def test_repeated_recovery_of_one_migration_still_gives_up(self, monkeypatch, tmp_path):
harness = _MigrateDeployHarness(
monkeypatch,
tmp_path,
[_p3018_stderr("20250329084805_new_cron_job_table")],
repeat_last=True,
confirmed_migrations=("20250329084805_new_cron_job_table",),
)
with pytest.raises(RuntimeError):
harness.run()
assert len(harness.deploy_calls) <= _ATTEMPT_BUDGET + 1
assert len(harness.deploy_calls) == 2
assert harness.resolved == ["20250329084805_new_cron_job_table"]
def test_timeouts_still_spend_the_budget(self, monkeypatch, tmp_path):
harness = _MigrateDeployHarness(
monkeypatch, tmp_path, ["timeout"], repeat_last=True
)
harness = _MigrateDeployHarness(monkeypatch, tmp_path, ["timeout"], repeat_last=True)
with pytest.raises(RuntimeError):
harness.run()
assert len(harness.deploy_calls) == _ATTEMPT_BUDGET
def test_a_baseline_that_never_lands_stops_after_the_budget(
self, monkeypatch, tmp_path
):
harness = _MigrateDeployHarness(
monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True
)
def test_an_unverified_baseline_stops_without_replaying_migrations(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
harness: Final = _MigrateDeployHarness(monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True)
harness.baseline_succeeds = False
with pytest.raises(RuntimeError):
harness.run()
assert len(harness.deploy_calls) == _ATTEMPT_BUDGET
assert len(harness.deploy_calls) == 1
def test_lock_contention_does_not_spend_the_failure_budget(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
harness: Final = _MigrateDeployHarness(
monkeypatch,
tmp_path,
["Error: P1002\nTimed out waiting for the advisory lock"] * 6 + ["ok"],
)
assert harness.run() is True
assert len(harness.deploy_calls) == 7
def test_duplicate_object_error_without_completion_proof_is_fatal(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
harness: Final = _MigrateDeployHarness(monkeypatch, tmp_path, [_p3018_stderr("20260101000000_x")])
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
harness.run()
assert harness.resolved == []
assert len(harness.deploy_calls) == 1
@pytest.mark.parametrize("name", ("20260101000000_x", "20260101000000_migration with spaces"))
def test_an_interrupted_migration_with_confirmed_sql_can_finish(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, name: str
) -> None:
harness: Final = _MigrateDeployHarness(
monkeypatch,
tmp_path,
[f"Error: P3009\nThe `{name}` migration failed", "ok"],
confirmed_migrations=(name,),
)
assert harness.run() is True
assert harness.resolved == [name]
def test_an_interrupted_migration_without_confirmation_stops(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
name: Final = "20260101000000_x"
started: Final = "2026-09-12 20:15:06.694553 UTC"
report: Final = f"Error: P3009\nThe `{name}` migration started at {started} failed"
harness: Final = _MigrateDeployHarness(
monkeypatch,
tmp_path,
[report],
)
with pytest.raises(RuntimeError, match="Migration completion could not be verified") as failure:
harness.run()
message: Final = str(failure.value)
assert name in message
assert started in message
assert "start record but no successful completion record" in message
assert "cannot determine whether its SQL committed" in message
assert "avoid repeating or skipping database changes" in message
assert "_prisma_migrations" in message
assert "migration.sql" in message
assert "same database" in message
assert "Only after verifying every migration change is present" in message
assert "prisma migrate resolve --applied <migration_name>" in message
assert "Only after verifying no migration changes remain" in message
assert "prisma migrate resolve --rolled-back <migration_name>" in message
assert "leave migration history unchanged" in message
assert "Repeated restarts alone" in message
assert report in message
assert len(harness.deploy_calls) == 1
assert harness.resolved == []
def test_an_unrecoverable_error_is_not_retried(self, monkeypatch, tmp_path):
harness = _MigrateDeployHarness(
monkeypatch,
tmp_path,
["Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near \"SLECT\"\n"],
['Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near "SLECT"\n'],
repeat_last=True,
)
@ -927,6 +1295,38 @@ class TestMigrateDeployAttemptAccounting:
assert harness.resolved == []
@pytest.mark.parametrize(
"steps,logs,script,expected",
(
(1, "", b"CREATE TABLE item (id int);", True),
(0, "", b"CREATE TABLE item (id int);", False),
(0, "already exists", b"CREATE TABLE item (id int);", False),
(1, "permission denied", b"CREATE TABLE item (id int);", False),
(1, "", b"CREATE TABLE item (id text);", False),
(2, "", b"CREATE TABLE item (id int);", False),
),
)
def test_migration_completion_requires_a_matching_successful_script(
steps: int, logs: str, script: bytes, expected: bool
) -> None:
import hashlib
from litellm_proxy_extras.migration_recovery import MigrationProgress
progress: Final = MigrationProgress(hashlib.sha256(b"CREATE TABLE item (id int);").hexdigest(), steps, logs)
assert progress.confirms_completion(script) is expected
def test_prisma_lock_waiting_has_its_own_deadline():
from litellm_proxy_extras.utils import _MigrateAttemptBudget
budget = _MigrateAttemptBudget(attempts_left=4, contention_seconds_left=2)
waiting = budget.after_contention(1)
assert waiting.attempts_left == 4
with pytest.raises(RuntimeError, match="advisory lock"):
waiting.after_contention(2)
class TestJWTKeyMappingCascade:
"""Regression tests for issue #33702.
@ -991,3 +1391,329 @@ class TestJWTKeyMappingCascade:
f"{path} must declare onDelete: Cascade on the JWT key mapping "
"relation (issue #33702)"
)
class TestStripPrismaQueryParams:
"""The psycopg URL the job connects with is derived from the Prisma-dialect
DATABASE_URL, whose TLS params mean something else to libpq."""
@staticmethod
def _query(url: str) -> dict[str, str]:
from urllib.parse import parse_qsl, urlparse
return dict(parse_qsl(urlparse(url).query))
@pytest.mark.parametrize(
"url,expected_user,expected_password,expected_host,expected_port,expected_sslmode",
(
(
"postgresql://u:wr ong'pw9@writer:5433/litellm",
"u",
"wr ong'pw9",
"writer",
"5433",
None,
),
(
"postgresql://u:wr ong'pw9@writer:5433/litellm?schema=x&sslmode=require",
"u",
"wr ong'pw9",
"writer",
"5433",
"require",
),
(
"postgresql://u:p%40ss%20w@writer:5433/litellm",
"u",
"p@ss w",
"writer",
"5433",
None,
),
(
"postgresql://u:p%z@writer:5433/litellm",
"u",
"p%z",
"writer",
"5433",
None,
),
(
"postgresql://u:wr ong'pw9@[::1]:5433/litellm",
"u",
"wr ong'pw9",
"::1",
"5433",
None,
),
),
)
def test_userinfo_is_encoded_for_libpq(
self,
url: str,
expected_user: str,
expected_password: str,
expected_host: str,
expected_port: str,
expected_sslmode: str | None,
) -> None:
from psycopg.conninfo import conninfo_to_dict
result: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
conninfo: Final = conninfo_to_dict(result)
assert conninfo["user"] == expected_user
assert conninfo["password"] == expected_password
assert conninfo["host"] == expected_host
assert conninfo["port"] == expected_port
assert conninfo["dbname"] == "litellm"
assert conninfo.get("sslmode") == expected_sslmode
assert "schema" not in conninfo
def test_empty_password_userinfo_keeps_its_separator(self) -> None:
url: Final = "postgresql://u:@writer/litellm"
assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url
def test_prisma_ca_sslcert_becomes_sslrootcert_with_verify_full(self) -> None:
url: Final = (
"postgresql://u:p@writer:5432/db?schema=public&sslmode=require&sslcert=/tmp/pinned.pem&sslaccept=strict"
)
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/tmp/pinned.pem"}
assert cleaned.startswith("postgresql://u:p@writer:5432/db?")
@pytest.mark.parametrize("sslmode", ["prefer", "require"])
@pytest.mark.parametrize("sslaccept", ["strict", "unknown-mode-prisma-treats-as-strict"])
def test_strict_verifies_chain_and_hostname_whatever_sslmode_prisma_was_given(
self, sslmode: str, sslaccept: str
) -> None:
url: Final = f"postgresql://writer/db?sslmode={sslmode}&sslcert=/certs/ca.pem&sslaccept={sslaccept}"
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/certs/ca.pem"}
def test_strict_with_tls_disabled_stays_off(self) -> None:
url: Final = "postgresql://writer/db?sslmode=disable&sslcert=/certs/ca.pem&sslaccept=strict"
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert self._query(cleaned) == {"sslmode": "disable"}
@pytest.mark.parametrize("sslaccept", ["&sslaccept=accept_invalid_certs", ""])
def test_without_strict_the_ca_is_dropped_so_libpq_checks_nothing_like_prisma(self, sslaccept: str) -> None:
url: Final = f"postgresql://writer/db?sslmode=require&sslcert=/certs/ca.pem{sslaccept}"
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert self._query(cleaned) == {"sslmode": "require"}
def test_a_ca_alone_without_strict_or_sslmode_leaves_libpq_its_defaults(self) -> None:
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params("postgresql://writer/db?sslcert=/certs/ca.pem")
assert cleaned == "postgresql://writer/db"
def test_a_libpq_client_certificate_pair_is_left_alone(self) -> None:
url: Final = (
"postgresql://writer/db?sslmode=verify-full&sslrootcert=/ca.pem&sslcert=/client.crt&sslkey=/client.key"
)
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert self._query(cleaned) == {
"sslmode": "verify-full",
"sslrootcert": "/ca.pem",
"sslcert": "/client.crt",
"sslkey": "/client.key",
}
def test_an_explicit_sslrootcert_wins_over_the_prisma_sslcert(self) -> None:
url: Final = "postgresql://writer/db?sslmode=require&sslrootcert=/ca.pem&sslcert=/pinned.pem&sslaccept=strict"
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/ca.pem"}
def test_prisma_only_params_are_dropped_and_plain_urls_pass_through(self) -> None:
url: Final = "postgresql://u:p@pooler:6543/db?schema=tenant&pgbouncer=true&connection_limit=5&connect_timeout=3"
cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert cleaned == "postgresql://u:p@pooler:6543/db?connect_timeout=3"
assert (
ProxyExtrasDBManager._strip_prisma_query_params("postgresql://u:p@writer/db")
== "postgresql://u:p@writer/db"
)
class TestBuildRequestLogIndexes:
"""The migration job hands the index build the direct database URL and the schema
the migrations target, waits for it, and reports its result."""
@pytest.fixture
def builds(self) -> SimpleQueue[tuple[str, str]]:
return SimpleQueue()
@pytest.fixture
def build(self, builds: SimpleQueue[tuple[str, str]]) -> Callable[[str, str], bool]:
def record(database_url: str, schema: str) -> bool:
builds.put((database_url, schema))
return True
return record
def test_the_build_gets_the_direct_url_without_prisma_params_and_the_prisma_schema(
self, monkeypatch: pytest.MonkeyPatch, builds: SimpleQueue[tuple[str, str]], build: Callable[[str, str], bool]
) -> None:
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@pooler:6543/db?schema=tenant&pgbouncer=true")
monkeypatch.setenv("DIRECT_URL", "postgresql://u:p@primary:5432/db?connection_limit=1")
assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True
assert builds.get_nowait() == ("postgresql://u:p@primary:5432/db", "tenant")
assert builds.empty()
def test_the_build_defaults_to_the_database_url_and_the_public_schema(
self, monkeypatch: pytest.MonkeyPatch, builds: SimpleQueue[tuple[str, str]], build: Callable[[str, str], bool]
) -> None:
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@primary:5432/db")
monkeypatch.delenv("DIRECT_URL", raising=False)
assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True
assert builds.get_nowait() == ("postgresql://u:p@primary:5432/db", "public")
assert builds.empty()
def test_a_build_that_leaves_indexes_missing_is_reported_so_the_job_reruns(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@primary:5432/db")
assert ProxyExtrasDBManager.build_request_log_indexes(build=lambda url, schema: False) is False
def test_without_a_database_url_nothing_is_built(
self, monkeypatch: pytest.MonkeyPatch, builds: SimpleQueue[tuple[str, str]], build: Callable[[str, str], bool]
) -> None:
monkeypatch.delenv("DATABASE_URL", raising=False)
assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True
assert builds.empty()
class TestStartRequestLogIndexBuild:
"""A serving proxy that ran the migrations starts the index build on a daemon thread
and goes on to serve while it runs."""
def test_the_build_runs_on_a_daemon_thread_that_does_not_hold_up_the_caller(self) -> None:
release: Final = threading.Event()
builds: Final[SimpleQueue[str]] = SimpleQueue()
def build() -> bool:
assert release.wait(5), "the caller never came back from start_request_log_index_build"
builds.put(threading.current_thread().name)
return True
thread: Final = ProxyExtrasDBManager.start_request_log_index_build(build=build)
assert builds.empty(), "the build ran before start_request_log_index_build returned"
assert thread.daemon is True
release.set()
thread.join(5)
assert builds.get_nowait() == "litellm-request-log-indexes"
assert builds.empty()
class TestRunMigrationJob:
"""`run_migration_job` is `setup_database` followed by the index build, each step's
result deciding whether the job reports success."""
@pytest.fixture
def calls(self) -> SimpleQueue[tuple[str, bool, bool] | tuple[str]]:
return SimpleQueue()
@pytest.fixture
def setup(
self, calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]]
) -> Callable[[bool], Callable[[bool, bool], bool]]:
def record(result: bool) -> Callable[[bool, bool], bool]:
def setup_database(use_migrate: bool, use_v2_resolver: bool) -> bool:
calls.put(("setup", use_migrate, use_v2_resolver))
return result
return setup_database
return record
@pytest.fixture
def build(self, calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]]) -> Callable[[bool], Callable[[], bool]]:
def record(result: bool) -> Callable[[], bool]:
def build_request_log_indexes() -> bool:
calls.put(("build",))
return result
return build_request_log_indexes
return record
def test_the_job_builds_the_indexes_after_the_migrations_succeed(
self,
calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]],
setup: Callable[[bool], Callable[[bool, bool], bool]],
build: Callable[[bool], Callable[[], bool]],
) -> None:
assert ProxyExtrasDBManager.run_migration_job(True, False, setup=setup(True), build=build(True)) is True
assert (calls.get_nowait(), calls.get_nowait()) == (("setup", True, False), ("build",))
assert calls.empty()
def test_the_job_fails_without_building_when_the_migrations_fail(
self,
calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]],
setup: Callable[[bool], Callable[[bool, bool], bool]],
build: Callable[[bool], Callable[[], bool]],
) -> None:
assert ProxyExtrasDBManager.run_migration_job(True, True, setup=setup(False), build=build(True)) is False
assert calls.get_nowait() == ("setup", True, True)
assert calls.empty()
def test_the_job_fails_when_an_index_could_not_be_built(
self,
calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]],
setup: Callable[[bool], Callable[[bool, bool], bool]],
build: Callable[[bool], Callable[[], bool]],
) -> None:
assert ProxyExtrasDBManager.run_migration_job(True, True, setup=setup(True), build=build(False)) is False
assert (calls.get_nowait(), calls.get_nowait()) == (("setup", True, True), ("build",))
assert calls.empty()
class TestMigrationJobOwnedDrift:
JOB_INDEXES: Final = (
"-- CreateIndex\n"
'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");\n'
"\n-- CreateIndex\n"
'CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");\n'
)
def test_a_plain_spend_logs_table_only_loses_the_migration_job_indexes(self) -> None:
filtered: Final = ProxyExtrasDBManager._filter_migration_job_owned_drift(
_PARTITIONED_DRIFT_SQL + self.JOB_INDEXES, partitioned=False
)
assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered
assert "LiteLLM_SpendLogs_api_key_startTime_idx" not in filtered
assert 'PRIMARY KEY ("request_id")' in filtered
def test_a_partitioned_spend_logs_table_also_loses_its_partitioning_artifacts(self) -> None:
filtered: Final = ProxyExtrasDBManager._filter_migration_job_owned_drift(
_PARTITIONED_DRIFT_SQL + self.JOB_INDEXES, partitioned=True
)
assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered
assert 'PRIMARY KEY ("request_id")' not in filtered
assert "LiteLLM_SpendLogs_legacy" not in filtered
assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered

View file

@ -0,0 +1,149 @@
import re
from pathlib import Path
from typing import Final
import pytest
from litellm_proxy_extras.migration_recovery import is_inert_migration
from litellm_proxy_extras.request_log_indexes import (
REQUEST_LOG_INDEXES,
RequestLogIndex,
filter_request_log_index_diff,
)
PACKAGE: Final = Path(__file__).resolve().parents[2] / "litellm-proxy-extras" / "litellm_proxy_extras"
SCHEMA: Final = PACKAGE / "schema.prisma"
INERT_MIGRATIONS: Final = (
"20260823000000_add_spend_logs_api_key_starttime_index",
"20260831120001_spend_logs_litellm_call_id_index",
)
CALL_ID_INDEX: Final = RequestLogIndex(
"LiteLLM_SpendLogs", "LiteLLM_SpendLogs_litellm_call_id_idx", '("litellm_call_id")'
)
def _prisma_indexes_of(schema: str, model: str) -> frozenset[str]:
"""The index names Prisma derives for a model's @@index declarations: <model>_<columns joined by _>_idx."""
body: Final = re.search(rf"model {model} \{{(.*?)\n\}}", schema, re.DOTALL)
assert body is not None, model
declarations: Final[tuple[str, ...]] = tuple(
match.group(1) for match in re.finditer(r"@@index\(\[([^\]]+)\]\)", body.group(1))
)
return frozenset(
f"{model}_{'_'.join(column.strip() for column in columns.split(','))}_idx" for columns in declarations
)
class TestTheIndexList:
def test_every_migration_job_index_is_declared_in_the_prisma_schema_under_the_same_name(self) -> None:
schema: Final = SCHEMA.read_text()
for index in REQUEST_LOG_INDEXES:
assert index.name in _prisma_indexes_of(schema, index.table), index
@pytest.mark.parametrize("name", INERT_MIGRATIONS)
def test_the_migrations_that_used_to_build_these_indexes_run_no_sql(self, name: str) -> None:
assert is_inert_migration((PACKAGE / "migrations" / name / "migration.sql").read_text())
class TestIsInertMigration:
@pytest.mark.parametrize(
"script",
(
"",
"-- only a comment\n",
"/* block */\n-- line\n",
"-- a semicolon; in a comment\n",
";\n;",
"-- why\nSELECT 1;\n",
"select 1",
),
ids=(
"empty",
"line-comment",
"both-comments",
"semicolon-in-comment",
"bare-separators",
"select-1",
"lowercase",
),
)
def test_comments_and_a_select_1_alone_are_inert(self, script: str) -> None:
assert is_inert_migration(script) is True
@pytest.mark.parametrize(
"script",
(
"SELECT 2;",
'SELECT 1 FROM "LiteLLM_SpendLogs";',
'-- comment\nCREATE INDEX "ix" ON "t" ("a");',
"/* c */ ALTER TABLE t ADD COLUMN a TEXT",
'SELECT 1; DROP INDEX "ix";',
),
ids=("select-2", "select-from", "index-after-comment", "alter-after-block-comment", "drop-after-select-1"),
)
def test_any_statement_is_not_inert(self, script: str) -> None:
assert is_inert_migration(script) is False
class TestPartitionIndexName:
def test_a_partition_gets_the_name_postgres_would_give_an_inherited_index(self) -> None:
assert CALL_ID_INDEX.partition_index_name("LiteLLM_SpendLogs_p2026_09") == (
"LiteLLM_SpendLogs_p2026_09_litellm_call_id_idx"
)
def test_an_index_not_prefixed_by_its_table_keeps_its_whole_name(self) -> None:
index: Final = RequestLogIndex("LiteLLM_SpendLogs", "call_id_lookup", '("litellm_call_id")')
assert index.partition_index_name("LiteLLM_SpendLogs_pdefault") == "LiteLLM_SpendLogs_pdefault_call_id_lookup"
def test_a_long_name_is_cut_to_63_bytes_with_a_digest_that_keeps_partitions_apart(self) -> None:
first: Final = CALL_ID_INDEX.partition_index_name("LiteLLM_SpendLogs_p" + "x" * 50 + "_2026_09")
second: Final = CALL_ID_INDEX.partition_index_name("LiteLLM_SpendLogs_p" + "x" * 50 + "_2026_10")
assert len(first.encode()) == 63 and len(second.encode()) == 63
assert first != second
assert first.startswith("LiteLLM_SpendLogs_p") and first[-9] == "_"
def test_the_byte_limit_counts_multibyte_characters(self) -> None:
name: Final = CALL_ID_INDEX.partition_index_name("é" * 40)
assert len(name.encode()) <= 63 and len(name) < 63
class TestColumns:
def test_the_columns_are_the_quoted_names_of_the_definition_in_order(self) -> None:
index: Final = RequestLogIndex(
"LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", '("api_key", "startTime")'
)
assert index.columns == ("api_key", "startTime")
def test_every_migration_job_index_names_at_least_one_column(self) -> None:
assert all(index.columns for index in REQUEST_LOG_INDEXES)
DRIFT_WITH_BOTH_INDEXES: Final = (
"-- CreateIndex\n"
'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");\n'
"\n"
"-- CreateIndex\n"
'CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");\n'
)
class TestFilterRequestLogIndexDiff:
def test_a_drift_script_that_only_creates_the_migration_job_indexes_becomes_empty(self) -> None:
assert filter_request_log_index_diff(DRIFT_WITH_BOTH_INDEXES) == ""
def test_other_statements_survive_with_the_migration_job_indexes_removed(self) -> None:
other: Final = '-- AlterTable\nALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;\n'
filtered: Final = filter_request_log_index_diff(other + DRIFT_WITH_BOTH_INDEXES)
assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered
assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered
assert "LiteLLM_SpendLogs_api_key_startTime_idx" not in filtered
def test_an_index_of_another_name_on_spend_logs_is_kept(self) -> None:
sql: Final = 'CREATE INDEX "LiteLLM_SpendLogs_end_user_idx" ON "LiteLLM_SpendLogs"("end_user");\n'
assert filter_request_log_index_diff(sql) == sql
def test_a_drop_of_a_migration_job_index_is_kept_for_the_operator_to_see(self) -> None:
sql: Final = 'DROP INDEX "LiteLLM_SpendLogs_litellm_call_id_idx";\n'
assert filter_request_log_index_diff(sql) == sql
def test_an_empty_script_stays_empty(self) -> None:
assert filter_request_log_index_diff("") == ""

View file

@ -5,7 +5,7 @@ import tempfile
from pathlib import Path
import pytest
from litellm_proxy_extras.utils import filter_hand_built_spend_logs_index_diff
from litellm_proxy_extras.request_log_indexes import filter_request_log_index_diff
@pytest.mark.skipif(
@ -17,7 +17,9 @@ def test_schema_migration_in_sync():
Applies every committed migration to an empty database, then diffs the result
against schema.prisma. A non-empty diff means the schema was changed without a
matching migration being generated.
matching migration being generated. The request-log indexes the migration job
builds are declared in the schema and deliberately absent from the migrations,
so those statements are filtered out before the diff is judged.
"""
db_url = os.environ["DATABASE_URL"]
source_migrations_dir = Path(
@ -60,12 +62,15 @@ def test_schema_migration_in_sync():
text=True,
)
if diff.returncode == 2 and filter_hand_built_spend_logs_index_diff(diff.stdout).strip():
pytest.fail(
"Schema changes detected that no migration captures. Run "
"`python litellm/ci_cd/run_migration.py <migration_name>`.\n\n"
+ diff.stdout
)
assert diff.returncode in (0, 2), f"prisma migrate diff errored: {diff.stderr}"
if diff.returncode == 2:
drift = filter_request_log_index_diff(diff.stdout)
if drift.strip():
pytest.fail(
"Schema changes detected that no migration captures. Run "
"`python litellm/ci_cd/run_migration.py <migration_name>`.\n\n"
+ drift
)
else:
assert diff.returncode == 0, f"prisma migrate diff errored: {diff.stderr}"
finally:
shutil.rmtree(temp_base, ignore_errors=True)

View file

@ -1,12 +1,15 @@
import os
import threading
import time
import uuid
from collections.abc import Iterator, Mapping
from types import MappingProxyType
from typing import Final
import pytest
from litellm_proxy_extras.utils import INDEX_REPAIR_ADVISORY_LOCK_KEY, ProxyExtrasDBManager
from litellm_proxy_extras.migration_lock import MIGRATION_LOCK_KEY
from litellm_proxy_extras.request_log_indexes import REQUEST_LOG_INDEXES
from litellm_proxy_extras.utils import INDEX_REPAIR_ADVISORY_LOCK_KEY, ProxyExtrasDBManager, _InvalidIndex
psycopg = pytest.importorskip("psycopg")
@ -20,6 +23,7 @@ requires_db: Final = pytest.mark.skipif(
HEALTH_TABLE: Final = "LiteLLM_HealthCheckTable"
HEALTH_INDEX: Final = "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx"
HEALTH_INDEX_COLUMNS: Final = '"model_id", "model_name", "checked_at" DESC'
SECOND_HEALTH_INDEX: Final = "LiteLLM_HealthCheckTable_model_name_idx"
LOOKALIKE_TABLE: Final = "LiteLLMLookalikeTable"
LOOKALIKE_INDEX: Final = "LiteLLMLookalikeTable_id_idx"
PARTITIONED_TABLE: Final = "LiteLLM_PartitionedTable"
@ -112,6 +116,48 @@ def test_repair_rebuilds_invalid_litellm_indexes_and_leaves_lookalike_tables_alo
assert _index_validity(scratch_schema) == {HEALTH_INDEX: True, LOOKALIKE_INDEX: False}
@requires_db
@pytest.mark.parametrize("partitioned", (False, True))
def test_startup_repair_defers_managed_indexes_but_repairs_other_indexes(
scratch_schema: str, partitioned: bool
) -> None:
table: Final = "LiteLLM_DifferentlyNamedLeaf" if partitioned else "LiteLLM_SpendLogs"
managed: Final = REQUEST_LOG_INDEXES[0]
name: Final = managed.partition_index_name(table) if partitioned else managed.name
with psycopg.connect(_base_url(), autocommit=True) as connection:
connection.execute(
f'CREATE TABLE "{scratch_schema}"."LiteLLM_SpendLogs" '
'(api_key TEXT, "startTime" TIMESTAMPTZ, litellm_call_id TEXT)'
+ (' PARTITION BY RANGE ("startTime")' if partitioned else '')
)
if partitioned:
connection.execute(
f'CREATE TABLE "{scratch_schema}"."{table}" PARTITION OF '
f'"{scratch_schema}"."LiteLLM_SpendLogs" FOR VALUES FROM (MINVALUE) TO (MAXVALUE)'
)
_leave_invalid_index(scratch_schema, table, name, managed.definition.strip("()"))
_leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
assert ProxyExtrasDBManager.repair_invalid_indexes(skip_request_log_indexes=True) is True
assert _index_validity(scratch_schema) == {name: False, HEALTH_INDEX: True}
assert ProxyExtrasDBManager.build_request_log_indexes() is True
assert all(_index_validity(scratch_schema).values())
@requires_db
def test_database_setup_leaves_managed_repair_to_the_request_log_builder(fresh_database: str) -> None:
assert ProxyExtrasDBManager.setup_database(use_migrate=True) is True
managed: Final = REQUEST_LOG_INDEXES[0]
_leave_invalid_index(fresh_database, managed.table, managed.name, managed.definition.strip("()"))
assert ProxyExtrasDBManager.setup_database(use_migrate=True) is True
assert _index_validity(fresh_database)[managed.name] is False
assert ProxyExtrasDBManager.build_request_log_indexes() is True
assert all(_index_validity(fresh_database)[index.name] for index in REQUEST_LOG_INDEXES)
@requires_db
def test_repair_drops_leftovers_of_interrupted_rebuilds(scratch_schema: str) -> None:
_leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
@ -167,7 +213,74 @@ def test_repair_yields_to_the_replica_holding_the_repair_lock(scratch_schema: st
@requires_db
def test_repair_gives_up_on_a_blocked_rebuild_and_finishes_it_on_the_next_startup(scratch_schema: str) -> None:
def test_repair_yields_to_the_migration_job_building_indexes_under_the_migration_lock(scratch_schema: str) -> None:
"""A migration job's index build holds the migration lock while its CREATE INDEX CONCURRENTLY
is cataloged as invalid; the repair must not rebuild that in-flight index."""
_leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
with psycopg.connect(_base_url(), autocommit=True) as index_builder:
index_builder.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,))
assert ProxyExtrasDBManager.repair_invalid_indexes() is False
assert _index_validity(scratch_schema) == {HEALTH_INDEX: False}
assert ProxyExtrasDBManager.repair_invalid_indexes() is True
assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
def _hold_migration_lock_once_free(release: threading.Event) -> None:
with psycopg.connect(_base_url(), autocommit=True) as resolver:
resolver.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,))
release.wait(timeout=60)
def _wait_until_a_session_queues_for_the_migration_lock() -> None:
with psycopg.connect(_base_url(), autocommit=True) as conn:
for _ in range(200):
queued: Final = conn.execute(
"SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND NOT granted "
"AND classid = %s AND objid = %s",
(MIGRATION_LOCK_KEY >> 32, MIGRATION_LOCK_KEY & 0xFFFFFFFF),
).fetchone()
if queued is not None and queued[0]:
return
time.sleep(0.05)
pytest.fail("no session queued for the migration lock")
@requires_db
def test_repair_releases_the_migration_lock_between_indexes_so_a_booting_resolver_gets_in(
scratch_schema: str,
) -> None:
"""A v2 resolver on another replica waits for the migration lock; with two invalid
indexes to rebuild it must get the lock after the first REINDEX, not after both."""
_leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
_leave_invalid_index(scratch_schema, HEALTH_TABLE, SECOND_HEALTH_INDEX, '"model_name"')
release: Final = threading.Event()
resolver: Final = threading.Thread(target=_hold_migration_lock_once_free, args=(release,))
def repair_then_let_a_resolver_queue_for_the_lock(
conn: "psycopg.Connection[tuple[str, str, str]]", index: _InvalidIndex
) -> None:
ProxyExtrasDBManager._repair_index(conn, index)
if not resolver.is_alive():
resolver.start()
_wait_until_a_session_queues_for_the_migration_lock()
try:
assert (
ProxyExtrasDBManager.repair_invalid_indexes(repair=repair_then_let_a_resolver_queue_for_the_lock) is False
)
assert sorted(_index_validity(scratch_schema).values()) == [False, True]
finally:
release.set()
resolver.join()
assert ProxyExtrasDBManager.repair_invalid_indexes() is True
assert _index_validity(scratch_schema) == {HEALTH_INDEX: True, SECOND_HEALTH_INDEX: True}
@requires_db
def test_repair_gives_up_on_a_blocked_rebuild_and_finishes_it_on_the_next_boot(scratch_schema: str) -> None:
_leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
with psycopg.connect(_base_url()) as pin:

View file

@ -0,0 +1,966 @@
import os
import queue
import shutil
import subprocess
import sys
import threading
import time
import uuid
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
import psycopg
import pytest
from litellm_proxy_extras import request_log_indexes
from litellm_proxy_extras.migration_lock import MIGRATION_LOCK_KEY, migration_lock
from litellm_proxy_extras.migration_recovery import roll_back_failed_inert_migration
from litellm_proxy_extras.request_log_indexes import (
REQUEST_LOG_INDEXES,
RequestLogIndex,
build_index_on_partitioned_table,
ensure_request_log_indexes,
)
from litellm_proxy_extras.utils import ProxyExtrasDBManager
from psycopg import sql
from psycopg.abc import Params, QueryNoTemplate
from psycopg.rows import class_row
pytestmark = pytest.mark.timeout(900)
requires_db: Final = pytest.mark.skipif(
"DATABASE_URL" not in os.environ,
reason="requires a postgres database (DATABASE_URL)",
)
REPO: Final = Path(__file__).resolve().parents[2]
PACKAGE: Final = REPO / "litellm-proxy-extras" / "litellm_proxy_extras"
PARTITION_SCRIPT: Final = REPO / "db_scripts" / "partition_spend_logs.sql"
API_KEY_INDEX_MIGRATION: Final = "20260823000000_add_spend_logs_api_key_starttime_index"
CALL_ID_INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index"
API_KEY_INDEX: Final = "LiteLLM_SpendLogs_api_key_startTime_idx"
CALL_ID_INDEX: Final = "LiteLLM_SpendLogs_litellm_call_id_idx"
PARTITIONED_PARENT_ERROR: Final = 'cannot create index on partitioned table "LiteLLM_SpendLogs" concurrently'
ORIGINAL_MIGRATION_SQL: Final = MappingProxyType(
{
API_KEY_INDEX_MIGRATION: (
"-- CreateIndex\n"
'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" '
'ON "LiteLLM_SpendLogs"("api_key", "startTime");\n'
),
CALL_ID_INDEX_MIGRATION: (
"-- CreateIndex\n"
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" '
'ON "LiteLLM_SpendLogs"("litellm_call_id");\n'
),
}
)
CALL_ID_INDEX_DEFINITION: Final = next(index for index in REQUEST_LOG_INDEXES if index.name == CALL_ID_INDEX)
RELEASES: Final = pytest.mark.parametrize(
"release", (API_KEY_INDEX_MIGRATION, CALL_ID_INDEX_MIGRATION), ids=("v1.102.1", "v1.103.0")
)
PARTITIONS: Final = MappingProxyType(
{
"LiteLLM_SpendLogs_p2026_08": ("2026-08-01", "2026-09-01"),
"LiteLLM_SpendLogs_p2026_09": ("2026-09-01", "2026-10-01"),
}
)
DEFAULT_PARTITION: Final = "LiteLLM_SpendLogs_pdefault"
ROWS_PER_PARTITION: Final = 200
RESOLVERS: Final = pytest.mark.parametrize("use_v2_resolver", (True, False), ids=("v2", "v1"))
def _base_url() -> str:
return os.environ["DATABASE_URL"].split("?")[0]
def _migrate_deploy(database_url: str, schema: Path) -> "subprocess.CompletedProcess[str]":
return subprocess.run(
[sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(schema)],
capture_output=True,
text=True,
env={**os.environ, "DATABASE_URL": database_url},
)
def _release_layout(prisma_dir: Path, before: str) -> Path:
"""The shipped migrations older than `before`, with the two index migrations written
the way the releases that shipped them did: the Prisma layout of a proxy on that release."""
(prisma_dir / "migrations").mkdir(parents=True)
shutil.copy(PACKAGE / "schema.prisma", prisma_dir / "schema.prisma")
for migration in sorted((PACKAGE / "migrations").iterdir()):
if migration.is_dir() and migration.name < before:
shutil.copytree(migration, prisma_dir / "migrations" / migration.name)
for name, original in ORIGINAL_MIGRATION_SQL.items():
if (prisma_dir / "migrations" / name).is_dir():
(prisma_dir / "migrations" / name / "migration.sql").write_text(original)
return prisma_dir / "schema.prisma"
def _deploy_release(database_url: str, prisma_dir: Path, before: str) -> None:
deployed: Final = _migrate_deploy(database_url, _release_layout(prisma_dir, before))
assert deployed.returncode == 0, deployed.stderr
def _insert_spend_log(
conn: "psycopg.Connection[tuple[object, ...]]", request_id: str, day: str, table: str = "LiteLLM_SpendLogs"
) -> None:
conn.execute(
sql.SQL(
'INSERT INTO {} ("request_id", "call_type", "startTime", "endTime", "api_key") VALUES (%s, %s, %s, %s, %s)'
).format(sql.Identifier(table)),
(request_id, "acompletion", day, day, f"key-{request_id[-1]}"),
)
def _partition_spend_logs(database_url: str) -> None:
with psycopg.connect(database_url, autocommit=True) as conn:
conn.execute(PARTITION_SCRIPT.read_bytes())
for partition, (start, stop) in PARTITIONS.items():
conn.execute(
sql.SQL('CREATE TABLE {} PARTITION OF "LiteLLM_SpendLogs" FOR VALUES FROM ({}) TO ({})').format(
sql.Identifier(partition), sql.Literal(start), sql.Literal(stop)
)
)
for row in range(ROWS_PER_PARTITION):
_insert_spend_log(conn, f"{partition}-{row}", start)
for row in range(ROWS_PER_PARTITION):
_insert_spend_log(conn, f"default-{row}", "2020-01-01")
@pytest.fixture
def release() -> str:
"""The first migration a database has not applied yet; the v1.103.0 shape unless a test parametrizes it."""
return CALL_ID_INDEX_MIGRATION
@pytest.fixture
def scratch_database(release: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[str]:
"""A deployment stopped before `release`, with DATABASE_URL pointed at it so
ProxyExtrasDBManager upgrades it like a booting proxy."""
admin_url: Final = _base_url()
name: Final = f"spend_logs_index_{uuid.uuid4().hex[:8]}"
with psycopg.connect(admin_url, autocommit=True) as conn:
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name)))
database_url: Final = f"{admin_url.rsplit('/', 1)[0]}/{name}"
try:
_deploy_release(database_url, tmp_path / "prisma", release)
monkeypatch.delenv("DIRECT_URL", raising=False)
monkeypatch.setenv("DATABASE_URL", database_url)
yield database_url
finally:
with psycopg.connect(admin_url, autocommit=True) as conn:
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(name)))
@pytest.fixture
def partitioned_database(scratch_database: str) -> str:
_partition_spend_logs(scratch_database)
return scratch_database
def _fail_the_call_id_migration_like_the_shipped_release(database_url: str, tmp_path: Path) -> None:
"""Boot the original v1.103.0 layout once: its CONCURRENTLY statement fails on the
partitioned parent and leaves the call_id ledger row unfinished."""
failed: Final = _migrate_deploy(database_url, _release_layout(tmp_path / "v1.103.0", "99999999999999"))
assert failed.returncode != 0 and PARTITIONED_PARENT_ERROR in failed.stderr, failed.stderr
assert _ledger(database_url)[CALL_ID_INDEX_MIGRATION] == (False, False)
@dataclass(frozen=True, slots=True)
class _IndexRow:
name: str
valid: bool
@dataclass(frozen=True, slots=True)
class _AttachedRow:
table: str
index: str
@dataclass(frozen=True, slots=True)
class _LedgerRow:
name: str
finished: bool
rolled_back: bool
@dataclass(frozen=True, slots=True)
class _OidRow:
name: str
oid: int
def _index_validity(database_url: str, suffix: str) -> Mapping[str, bool]:
"""index name -> indisvalid for every index ending in `suffix` on the SpendLogs parent or one of its partitions."""
with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_IndexRow)) as cursor:
rows: Final = cursor.execute(
"SELECT c.relname AS name, i.indisvalid AS valid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
"WHERE c.relname LIKE %s AND (i.indrelid = to_regclass('\"LiteLLM_SpendLogs\"') OR i.indrelid IN "
"(SELECT inhrelid FROM pg_inherits WHERE inhparent = to_regclass('\"LiteLLM_SpendLogs\"'))) "
"ORDER BY c.relname",
(f"%{suffix}",),
).fetchall()
return MappingProxyType({row.name: row.valid for row in rows})
def _attached_children(database_url: str, parent_index: str) -> frozenset[tuple[str, str]]:
"""(partition, child index) pairs attached under the parent index."""
with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_AttachedRow)) as cursor:
rows: Final = cursor.execute(
'SELECT t.relname AS "table", c.relname AS index FROM pg_inherits i '
"JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_index x ON x.indexrelid = c.oid "
"JOIN pg_class t ON t.oid = x.indrelid "
"WHERE i.inhparent = to_regclass(%s)",
(f'"{parent_index}"',),
).fetchall()
return frozenset((row.table, row.index) for row in rows)
@dataclass(frozen=True, slots=True)
class _TableRow:
name: str
def _indexed_table(database_url: str, index: str) -> "str | None":
with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_TableRow)) as cursor:
row: Final = cursor.execute(
"SELECT t.relname AS name FROM pg_index x JOIN pg_class t ON t.oid = x.indrelid "
"WHERE x.indexrelid = to_regclass(%s)",
(f'"{index}"',),
).fetchone()
return None if row is None else row.name
def _ledger(database_url: str) -> Mapping[str, tuple[bool, bool]]:
"""migration name -> (finished, rolled back) for the newest ledger row of each migration."""
with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_LedgerRow)) as cursor:
rows: Final = cursor.execute(
"SELECT DISTINCT ON (migration_name) migration_name AS name, finished_at IS NOT NULL AS finished, "
"rolled_back_at IS NOT NULL AS rolled_back FROM _prisma_migrations ORDER BY migration_name, started_at DESC"
).fetchall()
return MappingProxyType({row.name: (row.finished, row.rolled_back) for row in rows})
def _index_oids(database_url: str) -> Mapping[str, int]:
"""index name -> oid for every index on the SpendLogs parent or one of its partitions; a rebuild changes the oid."""
with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_OidRow)) as cursor:
rows: Final = cursor.execute(
"SELECT c.relname AS name, c.oid::int AS oid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
"WHERE i.indrelid = to_regclass('\"LiteLLM_SpendLogs\"') OR i.indrelid IN "
"(SELECT inhrelid FROM pg_inherits WHERE inhparent = to_regclass('\"LiteLLM_SpendLogs\"'))"
).fetchall()
return MappingProxyType({row.name: row.oid for row in rows})
def _migration_job(use_v2_resolver: bool) -> bool:
return ProxyExtrasDBManager.run_migration_job(use_migrate=True, use_v2_resolver=use_v2_resolver)
def _assert_no_pending_migrations(database_url: str) -> None:
status: Final = _migrate_deploy(database_url, PACKAGE / "schema.prisma")
assert status.returncode == 0 and "No pending migrations" in status.stdout, status.stdout + status.stderr
def _assert_every_ledger_row_is_finished(database_url: str) -> Mapping[str, tuple[bool, bool]]:
ledger: Final = _ledger(database_url)
assert ledger[API_KEY_INDEX_MIGRATION] == (True, False) and ledger[CALL_ID_INDEX_MIGRATION] == (True, False)
assert all(finished and not rolled_back for finished, rolled_back in ledger.values()), ledger
return ledger
def _expected_children(partitions: tuple[str, ...], suffix: str) -> frozenset[tuple[str, str]]:
return frozenset((partition, f"{partition}_{suffix}") for partition in partitions)
def _assert_index_covers_every_partition(database_url: str, parent_index: str, suffix: str) -> None:
partitions: Final = (*PARTITIONS, DEFAULT_PARTITION)
assert _index_validity(database_url, suffix) == {parent_index: True} | {f"{p}_{suffix}": True for p in partitions}
assert _attached_children(database_url, parent_index) == _expected_children(partitions, suffix)
@requires_db
@RESOLVERS
@RELEASES
def test_a_partitioned_spend_logs_upgrade_builds_both_indexes_per_partition_and_a_rerun_is_idempotent(
partitioned_database: str, use_v2_resolver: bool
) -> None:
assert _migration_job(use_v2_resolver) is True
_assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx")
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
ledger: Final = _assert_every_ledger_row_is_finished(partitioned_database)
_assert_no_pending_migrations(partitioned_database)
oids: Final = _index_oids(partitioned_database)
with psycopg.connect(partitioned_database, autocommit=True) as conn:
conn.execute(
'CREATE TABLE "LiteLLM_SpendLogs_p2026_10" PARTITION OF "LiteLLM_SpendLogs" '
"FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')"
)
inherited: Final = frozenset(
("LiteLLM_SpendLogs_p2026_10", f"LiteLLM_SpendLogs_p2026_10_{suffix}")
for suffix in ("api_key_startTime_idx", "litellm_call_id_idx")
)
attached: Final = _attached_children(partitioned_database, API_KEY_INDEX) | _attached_children(
partitioned_database, CALL_ID_INDEX
)
assert inherited <= attached, attached
assert _migration_job(use_v2_resolver) is True
assert _ledger(partitioned_database) == ledger
assert {name: oid for name, oid in _index_oids(partitioned_database).items() if name in oids} == oids
@requires_db
@RESOLVERS
@RELEASES
def test_a_plain_spend_logs_upgrade_builds_both_indexes_and_a_second_job_run_rebuilds_nothing(
scratch_database: str, use_v2_resolver: bool
) -> None:
with psycopg.connect(scratch_database, autocommit=True) as conn:
for row in range(ROWS_PER_PARTITION):
_insert_spend_log(conn, f"flat-{row}", "2026-09-01")
assert _migration_job(use_v2_resolver) is True
assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True}
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
_assert_every_ledger_row_is_finished(scratch_database)
_assert_no_pending_migrations(scratch_database)
oids: Final = _index_oids(scratch_database)
assert _migration_job(use_v2_resolver) is True
assert _index_oids(scratch_database) == oids
@requires_db
@RESOLVERS
def test_a_database_that_applied_the_original_migration_files_sees_no_pending_migrations_and_no_rebuild(
scratch_database: str, use_v2_resolver: bool, tmp_path: Path
) -> None:
"""A plain table upgraded on v1.103.0 applied both original files. The inert files in
this build must neither re-run nor fail those rows, and the migration job must keep the
indexes the migrations built."""
deployed: Final = _migrate_deploy(scratch_database, _release_layout(tmp_path / "v1.103.0", "99999999999999"))
assert deployed.returncode == 0, deployed.stderr
before: Final = _ledger(scratch_database)
assert before[API_KEY_INDEX_MIGRATION] == (True, False) and before[CALL_ID_INDEX_MIGRATION] == (True, False)
oids: Final = _index_oids(scratch_database)
assert {API_KEY_INDEX, CALL_ID_INDEX} <= set(oids)
_assert_no_pending_migrations(scratch_database)
assert _migration_job(use_v2_resolver) is True
assert _ledger(scratch_database) == before
assert _index_oids(scratch_database) == oids
@requires_db
@RESOLVERS
def test_a_failed_call_id_ledger_row_from_a_v1_103_boot_is_rolled_back_and_the_inert_file_applied(
partitioned_database: str, use_v2_resolver: bool, tmp_path: Path
) -> None:
_fail_the_call_id_migration_like_the_shipped_release(partitioned_database, tmp_path)
assert _migration_job(use_v2_resolver) is True
_assert_every_ledger_row_is_finished(partitioned_database)
_assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx")
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
_assert_no_pending_migrations(partitioned_database)
with psycopg.connect(partitioned_database) as conn:
rows: Final = conn.execute(
"SELECT finished_at IS NOT NULL, rolled_back_at IS NOT NULL FROM _prisma_migrations "
"WHERE migration_name = %s ORDER BY started_at",
(CALL_ID_INDEX_MIGRATION,),
).fetchall()
assert rows == [(False, True), (True, False)], rows
@requires_db
def test_a_failed_row_whose_migration_still_runs_sql_in_this_build_is_left_for_the_operator(
partitioned_database: str, tmp_path: Path
) -> None:
_fail_the_call_id_migration_like_the_shipped_release(partitioned_database, tmp_path)
still_building: Final = tmp_path / "edited" / CALL_ID_INDEX_MIGRATION / "migration.sql"
still_building.parent.mkdir(parents=True)
still_building.write_text(ORIGINAL_MIGRATION_SQL[CALL_ID_INDEX_MIGRATION])
with migration_lock(partitioned_database) as coordinator:
assert roll_back_failed_inert_migration(coordinator, "public", still_building) is False
assert _ledger(partitioned_database)[CALL_ID_INDEX_MIGRATION] == (False, False)
@requires_db
def test_a_migration_without_a_failed_row_is_not_touched(partitioned_database: str) -> None:
inert: Final = PACKAGE / "migrations" / CALL_ID_INDEX_MIGRATION / "migration.sql"
before: Final = _ledger(partitioned_database)
with migration_lock(partitioned_database) as coordinator:
assert roll_back_failed_inert_migration(coordinator, "public", inert) is False
assert _ledger(partitioned_database) == before
def _pin_a_snapshot_on(database_url: str, table: str) -> "psycopg.Connection[tuple[object, ...]]":
pin: Final = psycopg.connect(database_url)
pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
pin.execute(sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table)))
return pin
def _leave_an_invalid_index(database_url: str, name: str, table: str, column: str) -> None:
with _pin_a_snapshot_on(database_url, table):
with psycopg.connect(database_url, autocommit=True) as builder:
builder.execute("SET statement_timeout = '1s'")
with pytest.raises(psycopg.errors.QueryCanceled):
builder.execute(
sql.SQL("CREATE INDEX CONCURRENTLY {} ON {} ({})").format(
sql.Identifier(name), sql.Identifier(table), sql.Identifier(column)
)
)
@requires_db
def test_an_invalid_index_of_the_managed_name_on_a_plain_table_is_rebuilt(scratch_database: str) -> None:
_leave_an_invalid_index(scratch_database, CALL_ID_INDEX, "LiteLLM_SpendLogs", "litellm_call_id")
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: False}
assert ensure_request_log_indexes(scratch_database, "public") is True
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
def _rebuild_as_another_replica(database_url: str, name: str, table: str, column: str) -> int:
"""Drop and rebuild the index from a second connection, as a replica that won the
race would, and return the oid of the index it built."""
with psycopg.connect(database_url, autocommit=True) as other_replica:
other_replica.execute(sql.SQL("DROP INDEX {}").format(sql.Identifier(name)))
other_replica.execute(
sql.SQL("CREATE INDEX {} ON {} ({})").format(
sql.Identifier(name), sql.Identifier(table), sql.Identifier(column)
)
)
return _index_oids(database_url)[name]
def _connecting_with_another_replica_acting_first(
statement: str, other_replica: Callable[[QueryNoTemplate], None]
) -> Callable[[str], "psycopg.Connection[tuple[object, ...]]"]:
"""A connect function whose cursors let `other_replica` act, once, right before the
first statement containing `statement` runs: the interleaving two replicas booting
together can produce, made deterministic."""
raced: Final = threading.Event()
class _RacedCursor(psycopg.Cursor[tuple[object, ...]]):
def execute( # pyright: ignore[reportIncompatibleMethodOverride] # the builder never runs a Template query
self,
query: QueryNoTemplate,
params: "Params | None" = None,
*,
prepare: "bool | None" = None,
binary: "bool | None" = None,
) -> "_RacedCursor":
text: Final = query.as_string(self.connection) if isinstance(query, sql.Composable) else query
if isinstance(text, str) and statement in text and not raced.is_set():
raced.set()
other_replica(query)
return super().execute(query, params, prepare=prepare, binary=binary)
def connect(database_url: str) -> "psycopg.Connection[tuple[object, ...]]":
return psycopg.connect(database_url, autocommit=True, cursor_factory=_RacedCursor)
return connect
@requires_db
def test_an_index_another_replica_made_valid_before_the_lock_was_taken_is_kept(scratch_database: str) -> None:
"""Two replicas boot against the same invalid index. The one that takes the lock
second must read the catalog again under it, or it drops the valid index the first
one just finished and starts the whole build over."""
_leave_an_invalid_index(scratch_database, CALL_ID_INDEX, "LiteLLM_SpendLogs", "litellm_call_id")
theirs: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
connect: Final = _connecting_with_another_replica_acting_first(
"pg_try_advisory_lock",
lambda _: theirs.put(
_rebuild_as_another_replica(scratch_database, CALL_ID_INDEX, "LiteLLM_SpendLogs", "litellm_call_id")
),
)
assert ensure_request_log_indexes(scratch_database, "public", (CALL_ID_INDEX_DEFINITION,), connect) is True
assert _index_oids(scratch_database)[CALL_ID_INDEX] == theirs.get_nowait()
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
@requires_db
def test_a_child_index_another_replica_attached_first_is_not_attached_twice(partitioned_database: str) -> None:
"""A replica that reaches the attach step after another one attached the same child
relies on ATTACH PARTITION being a no-op for an index already under that parent
(PostgreSQL 14 ALTER INDEX, ATExecAttachPartitionIdx, checked 2026-10-01); this test
is where that would surface if a future version or a code change made it an error."""
def attach_as_another_replica(statement: QueryNoTemplate) -> None:
with psycopg.connect(partitioned_database, autocommit=True) as other_replica:
other_replica.execute(statement)
connect: Final = _connecting_with_another_replica_acting_first("ATTACH PARTITION", attach_as_another_replica)
assert ensure_request_log_indexes(partitioned_database, "public", (CALL_ID_INDEX_DEFINITION,), connect) is True
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_an_invalid_child_index_left_by_an_interrupted_build_is_rebuilt_and_attached(
partitioned_database: str,
) -> None:
partition: Final = "LiteLLM_SpendLogs_p2026_08"
child: Final = f"{partition}_litellm_call_id_idx"
_leave_an_invalid_index(partitioned_database, child, partition, "litellm_call_id")
assert _index_validity(partitioned_database, "litellm_call_id_idx") == {child: False}
assert ensure_request_log_indexes(partitioned_database, "public") is True
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_partitioning_a_table_the_index_job_already_covered_moves_its_call_id_index_aside(scratch_database: str) -> None:
"""The runbook renames the Prisma indexes off the legacy table so the partitioned table can take their
names. The call-id index is built by the index job, not Prisma, and a deployment that partitions after
that job ran kept the name on the legacy table, so every later job run reported it as on another table."""
assert ensure_request_log_indexes(scratch_database, "public") is True
assert _indexed_table(scratch_database, CALL_ID_INDEX) == "LiteLLM_SpendLogs"
_partition_spend_logs(scratch_database)
assert _indexed_table(scratch_database, "LiteLLM_SpendLogs_legacy_litellm_call_id_idx") == "LiteLLM_SpendLogs_legacy"
assert ensure_request_log_indexes(scratch_database, "public") is True
_assert_index_covers_every_partition(scratch_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_an_index_of_that_name_on_another_table_is_left_alone_and_reported(partitioned_database: str) -> None:
with psycopg.connect(partitioned_database, autocommit=True) as conn:
conn.execute(f'CREATE INDEX "{CALL_ID_INDEX}" ON "LiteLLM_ErrorLogs" ("request_id")')
assert ensure_request_log_indexes(partitioned_database, "public") is False
assert _index_validity(partitioned_database, "litellm_call_id_idx") == {}
assert _indexed_table(partitioned_database, CALL_ID_INDEX) == "LiteLLM_ErrorLogs"
_assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx")
@requires_db
def test_an_invalid_index_of_a_child_name_on_another_table_is_not_dropped(partitioned_database: str) -> None:
child: Final = "LiteLLM_SpendLogs_p2026_08_litellm_call_id_idx"
_leave_an_invalid_index(partitioned_database, child, "LiteLLM_ErrorLogs", "request_id")
assert ensure_request_log_indexes(partitioned_database, "public") is False
assert _indexed_table(partitioned_database, child) == "LiteLLM_ErrorLogs"
assert _attached_children(partitioned_database, CALL_ID_INDEX) == frozenset()
@requires_db
def test_a_process_holding_the_migration_lock_makes_the_build_wait_for_the_next_job_run(scratch_database: str) -> None:
with psycopg.connect(scratch_database, autocommit=True) as other_replica:
other_replica.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,))
assert ensure_request_log_indexes(scratch_database, "public") is False
assert _index_validity(scratch_database, "litellm_call_id_idx") == {}
assert ensure_request_log_indexes(scratch_database, "public") is True
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
@requires_db
@RESOLVERS
def test_a_migration_job_that_could_not_build_the_indexes_reports_failure_and_succeeds_when_rerun(
scratch_database: str, use_v2_resolver: bool
) -> None:
"""The migration job waits for the build and exits by run_migration_job's result; a job
that exits 0 with the indexes missing would leave the table unindexed until the next
deploy or until a serving proxy's background build gets to them."""
with psycopg.connect(scratch_database, autocommit=True) as other_replica:
other_replica.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,))
assert _migration_job(use_v2_resolver) is False
_assert_every_ledger_row_is_finished(scratch_database)
assert _index_validity(scratch_database, "litellm_call_id_idx") == {}
assert _migration_job(use_v2_resolver) is True
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True}
@requires_db
@RESOLVERS
def test_the_serving_proxy_setup_applies_the_inert_migrations_and_builds_no_index(
partitioned_database: str, use_v2_resolver: bool
) -> None:
"""setup_database alone applies the inert files and builds nothing, so a serving proxy's
readiness is never held up by an index build; the build it starts afterwards, or the
migration job, is what puts the indexes in place."""
api_key_index_before: Final = _index_validity(partitioned_database, "api_key_startTime_idx")
assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
_assert_every_ledger_row_is_finished(partitioned_database)
_assert_no_pending_migrations(partitioned_database)
assert _index_validity(partitioned_database, "litellm_call_id_idx") == {}
assert _index_validity(partitioned_database, "api_key_startTime_idx") == api_key_index_before
assert _migration_job(use_v2_resolver) is True
_assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx")
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_a_role_that_may_not_create_indexes_is_logged_and_left_for_the_next_job_run(
scratch_database: str, caplog: pytest.LogCaptureFixture
) -> None:
with psycopg.connect(scratch_database, autocommit=True) as conn:
conn.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC")
conn.execute("CREATE ROLE spend_logs_reader LOGIN PASSWORD 'reader'")
conn.execute("GRANT USAGE ON SCHEMA public TO spend_logs_reader")
conn.execute('GRANT SELECT ON "LiteLLM_SpendLogs" TO spend_logs_reader')
reader_url: Final = scratch_database.replace("postgres:postgres@", "spend_logs_reader:reader@", 1)
try:
with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
assert ensure_request_log_indexes(reader_url, "public") is False
finally:
with psycopg.connect(scratch_database, autocommit=True) as conn:
conn.execute("DROP OWNED BY spend_logs_reader")
conn.execute("DROP ROLE spend_logs_reader")
assert "leaving them for the next index build" in caplog.text
assert _index_validity(scratch_database, "litellm_call_id_idx") == {}
@requires_db
def test_inserts_keep_flowing_while_the_partition_indexes_build(partitioned_database: str) -> None:
"""With a write open on one partition, the parent index goes on ONLY the parent and
the CONCURRENTLY child build waits for that write without blocking new INSERTs. A
plain CREATE INDEX on the parent would wait for the same write while holding SHARE
on the parent, queueing every new INSERT behind it."""
outcome: Final[list[bool]] = [] # mutable-ok: the builder thread hands its result back through it
with psycopg.connect(partitioned_database) as writer:
_insert_spend_log(writer, "LiteLLM_SpendLogs_p2026_08-open", "2026-08-15", table="LiteLLM_SpendLogs_p2026_08")
builder_thread: Final = threading.Thread(
target=lambda: outcome.append(_build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION))
)
builder_thread.start()
try:
_wait_until_the_build_is_waiting(partitioned_database)
with psycopg.connect(partitioned_database, autocommit=True) as late_writer:
late_writer.execute("SET lock_timeout = '1s'")
_insert_spend_log(late_writer, "LiteLLM_SpendLogs_p2026_08-late", "2026-08-16")
finally:
writer.commit()
builder_thread.join()
assert outcome == [True]
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
def _insert_for(database_url: str, seconds: float) -> None:
with psycopg.connect(database_url, autocommit=True) as conn:
conn.execute("SET lock_timeout = '1s'")
deadline: Final = time.monotonic() + seconds
while time.monotonic() < deadline:
_insert_spend_log(conn, f"lock-test-{uuid.uuid4().hex}", "2026-08-16")
time.sleep(0.05)
def _wait_for_blocked_ddl(database_url: str, query_pattern: str) -> bool:
with psycopg.connect(database_url, autocommit=True) as conn:
deadline: Final = time.monotonic() + 10
while time.monotonic() < deadline:
if conn.execute(
"SELECT 1 FROM pg_stat_activity WHERE wait_event_type = 'Lock' AND query ILIKE %s",
(query_pattern,),
).fetchone():
return True
time.sleep(0.01)
return False
@requires_db
def test_inserts_are_never_held_back_while_the_parent_index_waits_for_an_open_write(
partitioned_database: str,
) -> None:
outcome: Final[list[bool]] = [] # mutable-ok: the builder thread hands its result back through it
with psycopg.connect(partitioned_database) as writer:
_insert_spend_log(writer, "parent-index-lock-owner", "2026-08-15")
builder_thread: Final = threading.Thread(
target=lambda: outcome.append(_build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION))
)
builder_thread.start()
try:
assert _wait_for_blocked_ddl(partitioned_database, "%CREATE INDEX%ON ONLY%")
_insert_for(partitioned_database, 3)
finally:
try:
writer.commit()
finally:
builder_thread.join()
assert outcome == [True]
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_inserts_are_never_held_back_while_attach_partition_waits_for_a_reader_of_the_child_index(
partitioned_database: str,
) -> None:
partition: Final = "LiteLLM_SpendLogs_p2026_08"
child_index: Final = CALL_ID_INDEX_DEFINITION.partition_index_name(partition)
assert child_index == "LiteLLM_SpendLogs_p2026_08_litellm_call_id_idx"
with psycopg.connect(partitioned_database, autocommit=True) as conn:
conn.execute(
'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON ONLY "LiteLLM_SpendLogs" ("litellm_call_id")'
)
conn.execute(
sql.SQL('CREATE INDEX {} ON {} ("litellm_call_id")').format(
sql.Identifier(CALL_ID_INDEX_DEFINITION.partition_index_name(partition)), sql.Identifier(partition)
)
)
outcome: Final[list[bool]] = [] # mutable-ok: the builder thread hands its result back through it
with psycopg.connect(partitioned_database) as reader:
reader.execute("SET enable_seqscan = off")
reader.execute(
sql.SQL('SELECT count(*) FROM {} WHERE "litellm_call_id" IS NULL').format(sql.Identifier(partition))
).fetchone()
reader_pid: Final = reader.execute("SELECT pg_backend_pid()").fetchone()[0]
with psycopg.connect(partitioned_database, autocommit=True) as inspector:
child_lock: Final = inspector.execute(
"SELECT 1 FROM pg_locks WHERE pid = %s AND relation = to_regclass(%s) "
"AND mode = 'AccessShareLock' AND granted",
(reader_pid, f'"{child_index}"'),
).fetchone()
assert child_lock is not None
builder_thread: Final = threading.Thread(
target=lambda: outcome.append(_build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION))
)
builder_thread.start()
try:
assert _wait_for_blocked_ddl(partitioned_database, "%ATTACH PARTITION%")
_insert_for(partitioned_database, 3)
finally:
try:
reader.commit()
finally:
builder_thread.join()
assert outcome == [True]
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_a_parent_index_that_never_gets_its_lock_is_left_for_the_next_index_build(
partitioned_database: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
monkeypatch.setattr(request_log_indexes, "_DDL_LOCK_ATTEMPTS", 2)
with psycopg.connect(partitioned_database) as writer:
_insert_spend_log(writer, "parent-index-lock-owner", "2026-08-15")
with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is False
assert "leaving it for the next index build" in caplog.text
assert _indexed_table(partitioned_database, CALL_ID_INDEX) is None
writer.commit()
assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is True
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
@requires_db
def test_an_attach_that_never_gets_its_lock_is_left_for_the_next_index_build(
partitioned_database: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
partition: Final = "LiteLLM_SpendLogs_p2026_08"
child_index: Final = CALL_ID_INDEX_DEFINITION.partition_index_name(partition)
assert child_index == "LiteLLM_SpendLogs_p2026_08_litellm_call_id_idx"
with psycopg.connect(partitioned_database, autocommit=True) as conn:
conn.execute(
'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON ONLY "LiteLLM_SpendLogs" ("litellm_call_id")'
)
conn.execute(
sql.SQL('CREATE INDEX {} ON {} ("litellm_call_id")').format(
sql.Identifier(child_index), sql.Identifier(partition)
)
)
monkeypatch.setattr(request_log_indexes, "_DDL_LOCK_ATTEMPTS", 2)
with psycopg.connect(partitioned_database) as reader:
reader.execute("SET enable_seqscan = off")
reader.execute(
sql.SQL('SELECT count(*) FROM {} WHERE "litellm_call_id" IS NULL').format(sql.Identifier(partition))
).fetchone()
reader_pid: Final = reader.execute("SELECT pg_backend_pid()").fetchone()[0]
with psycopg.connect(partitioned_database, autocommit=True) as inspector:
child_lock: Final = inspector.execute(
"SELECT 1 FROM pg_locks WHERE pid = %s AND relation = to_regclass(%s) "
"AND mode = 'AccessShareLock' AND granted",
(reader_pid, f'"{child_index}"'),
).fetchone()
assert child_lock is not None
with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is False
assert "Could not get the lock for attaching" in caplog.text
reader.commit()
assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is True
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
def _build_in_its_own_session(database_url: str, index: RequestLogIndex) -> bool:
with psycopg.connect(database_url, autocommit=True) as builder:
return build_index_on_partitioned_table(builder, "public", index)
def _wait_until_the_build_is_waiting(database_url: str) -> None:
deadline: Final = time.monotonic() + 30
with psycopg.connect(database_url, autocommit=True) as conn:
while time.monotonic() < deadline:
waiting = conn.execute(
"SELECT 1 FROM pg_stat_activity WHERE query LIKE 'CREATE INDEX%' AND wait_event_type IS NOT NULL"
).fetchone()
if waiting is not None:
return
time.sleep(0.05)
pytest.fail("the partition index build never started waiting on the open write")
def _create_index(database_url: str, name: str, table: str, columns: str) -> int:
"""Create a plain index by hand, the way an operator's workaround would, and return its oid."""
with psycopg.connect(database_url, autocommit=True) as conn:
conn.execute(
sql.SQL("CREATE INDEX {} ON {} {}").format(sql.Identifier(name), sql.Identifier(table), sql.SQL(columns))
)
return _index_oids(database_url)[name]
@requires_db
@pytest.mark.parametrize(
("definition", "matches"),
(
(None, False),
('("litellm_call_id", "api_key")', True),
('("litellm_call_id") INCLUDE ("api_key")', False),
),
ids=("missing", "equivalent", "included-column"),
)
def test_equivalent_index_lookup_works_without_postgres_11_catalog_columns(
definition: str | None, matches: bool
) -> None:
schema: Final = f"index_catalog_{uuid.uuid4().hex}"
target: Final = RequestLogIndex("LiteLLM_SpendLogs", "managed_lookup", '("litellm_call_id", "api_key")')
with psycopg.connect(_base_url(), autocommit=True) as conn:
if definition is not None and "INCLUDE" in definition and conn.info.server_version < 110000:
pytest.skip("INCLUDE indexes require PostgreSQL 11")
conn.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
try:
conn.execute(
sql.SQL('CREATE TABLE {}."LiteLLM_SpendLogs" (litellm_call_id text, api_key text)').format(
sql.Identifier(schema)
)
)
if definition is not None:
conn.execute(
sql.SQL('CREATE INDEX operator_lookup ON {}."LiteLLM_SpendLogs" {}').format(
sql.Identifier(schema), sql.SQL(definition)
)
)
found: Final = request_log_indexes._equivalent_indexes(conn, schema, target.table, target.name, target)
assert tuple((index.name, index.valid) for index in found) == (
(("operator_lookup", True),) if matches else ()
)
finally:
conn.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
@requires_db
def test_a_valid_index_of_the_same_definition_under_another_name_is_renamed_instead_of_rebuilt(
scratch_database: str,
) -> None:
hand_built: Final = _create_index(scratch_database, "call_id_by_hand", "LiteLLM_SpendLogs", '("litellm_call_id")')
assert ensure_request_log_indexes(scratch_database, "public") is True
oids: Final = _index_oids(scratch_database)
assert "call_id_by_hand" not in oids and oids[CALL_ID_INDEX] == hand_built
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
@requires_db
def test_a_hand_built_child_index_under_another_name_is_renamed_and_attached(partitioned_database: str) -> None:
partition: Final = "LiteLLM_SpendLogs_p2026_08"
hand_built: Final = _create_index(
partitioned_database, "p2026_08_call_id_by_hand", partition, '("litellm_call_id")'
)
assert ensure_request_log_indexes(partitioned_database, "public") is True
_assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx")
oids: Final = _index_oids(partitioned_database)
assert "p2026_08_call_id_by_hand" not in oids and oids[f"{partition}_litellm_call_id_idx"] == hand_built
@requires_db
def test_an_index_with_another_definition_is_not_taken_for_the_managed_one(scratch_database: str) -> None:
with psycopg.connect(scratch_database, autocommit=True) as conn:
conn.execute(sql.SQL("DROP INDEX {}").format(sql.Identifier(API_KEY_INDEX)))
others: Final = {
"time_then_key": _create_index(
scratch_database, "time_then_key", "LiteLLM_SpendLogs", '("startTime", "api_key")'
),
"call_id_desc": _create_index(
scratch_database, "call_id_desc", "LiteLLM_SpendLogs", '("litellm_call_id" DESC)'
),
"call_id_then_key": _create_index(
scratch_database, "call_id_then_key", "LiteLLM_SpendLogs", '("litellm_call_id", "api_key")'
),
"call_id_pattern": _create_index(
scratch_database, "call_id_pattern", "LiteLLM_SpendLogs", '("litellm_call_id" text_pattern_ops)'
),
}
assert ensure_request_log_indexes(scratch_database, "public") is True
oids: Final = _index_oids(scratch_database)
assert {name: oids[name] for name in others} == others
assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True}
assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True}
@requires_db
def test_a_valid_partitioned_parent_index_under_another_name_is_renamed_with_its_children_kept(
partitioned_database: str, caplog: pytest.LogCaptureFixture
) -> None:
hand_built: Final = _create_index(
partitioned_database, "call_id_parent_by_hand", "LiteLLM_SpendLogs", '("litellm_call_id")'
)
children_before: Final = _attached_children(partitioned_database, "call_id_parent_by_hand")
with caplog.at_level("INFO", logger="litellm_proxy_extras"):
assert ensure_request_log_indexes(partitioned_database, "public") is True
assert "Building index" not in caplog.text
oids: Final = _index_oids(partitioned_database)
assert "call_id_parent_by_hand" not in oids and oids[CALL_ID_INDEX] == hand_built
assert _attached_children(partitioned_database, CALL_ID_INDEX) == children_before
assert _index_validity(partitioned_database, "litellm_call_id_idx")[CALL_ID_INDEX] is True
@requires_db
def test_a_second_copy_of_a_managed_index_is_reported_with_its_drop_statement_and_left_in_place(
scratch_database: str, caplog: pytest.LogCaptureFixture
) -> None:
assert ensure_request_log_indexes(scratch_database, "public") is True
copy: Final = _create_index(scratch_database, "call_id_copy", "LiteLLM_SpendLogs", '("litellm_call_id")')
with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
assert ensure_request_log_indexes(scratch_database, "public") is True
assert 'remove it with: DROP INDEX CONCURRENTLY "public"."call_id_copy"' in caplog.text
assert _index_oids(scratch_database)["call_id_copy"] == copy

View file

@ -3,6 +3,7 @@ import os
import signal
import sys
import urllib.parse
from collections.abc import Callable
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
@ -454,3 +455,21 @@ def test_db_push_without_the_prisma_runner_fails_the_migration_instead_of_crashi
assert PrismaManager.setup_database(use_migrate=False) is False
assert fake_prisma_cli.calls == []
@pytest.mark.parametrize(
("run", "outcome"),
(
(PrismaManager.build_request_log_indexes, False),
(PrismaManager.start_request_log_index_build, None),
),
ids=("wait-for-the-build", "start-the-build"),
)
def test_without_proxy_extras_the_index_build_reports_failure_instead_of_raising(
monkeypatch: pytest.MonkeyPatch, run: Callable[[], bool | None], outcome: bool | None
) -> None:
"""The migration job exits non-zero and a serving proxy keeps booting when the extras
package that owns the index build is not installed."""
monkeypatch.setitem(sys.modules, "litellm_proxy_extras.utils", None)
assert run() is outcome

View file

@ -2,14 +2,14 @@ import inspect
import os
from contextlib import nullcontext
from pathlib import Path
from queue import SimpleQueue
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import click
import fastapi
import pytest
import builtins
import types
import urllib.parse as urlparse
@ -19,6 +19,7 @@ import yaml
from uvicorn.config import LOOP_FACTORIES
from uvicorn.importer import import_from_string
from litellm.proxy import proxy_cli
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
@ -1924,19 +1925,29 @@ class TestQueryEngineReaperWiring:
class TestRunServerDbSetup:
"""Tests for run_server's prisma setup_database behavior."""
@pytest.fixture
def mock_setup_database(self) -> MagicMock:
return MagicMock()
@pytest.fixture
def database_startup(self, mock_setup_database: MagicMock) -> proxy_cli._DatabaseStartupDependencies:
return proxy_cli._DatabaseStartupDependencies(
setup=mock_setup_database, build=lambda: True, start=lambda: None
)
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
@patch("litellm.proxy.db.check_migration.check_prisma_schema_diff")
@patch("litellm.proxy.db.prisma_client.should_update_prisma_schema")
def test_use_prisma_db_push_flag_behavior(
self,
mock_should_update_schema,
mock_check_schema_diff,
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
):
mock_should_update_schema: MagicMock,
mock_check_schema_diff: MagicMock,
mock_atexit_register: MagicMock,
mock_subprocess_run: MagicMock,
mock_setup_database: MagicMock,
database_startup: proxy_cli._DatabaseStartupDependencies,
) -> None:
"""Test that use_prisma_db_push flag correctly controls PrismaManager.setup_database use_migrate parameter"""
from litellm.proxy.proxy_cli import run_server
@ -1946,19 +1957,17 @@ class TestRunServerDbSetup:
# Mock should_update_prisma_schema to return True (so setup_database gets called)
mock_should_update_schema.return_value = True
mock_proxy_module = MagicMock(
mock_proxy_module: Final = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {
k: v
for k, v in os.environ.items()
if k not in ("DATABASE_URL", "DIRECT_URL")
clean_env: Final = {
**{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")},
"DATABASE_URL": "postgresql://test:test@localhost:5432/test",
}
clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test"
with (
patch.dict(os.environ, clean_env, clear=True),
@ -1985,10 +1994,8 @@ class TestRunServerDbSetup:
# Test 1: Without --use_prisma_db_push flag (default behavior)
# use_prisma_db_push should be False (default), so use_migrate should be True
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False)
mock_setup_database.assert_called_with(
use_migrate=True, use_v2_resolver=False
)
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup)
mock_setup_database.assert_called_with(use_migrate=True, use_v2_resolver=False)
# Reset mocks
mock_setup_database.reset_mock()
@ -2000,43 +2007,44 @@ class TestRunServerDbSetup:
run_server.main(
["--local", "--skip_server_startup", "--use_prisma_db_push"],
standalone_mode=False,
obj=database_startup,
)
mock_setup_database.assert_called_with(
use_migrate=False, use_v2_resolver=False
)
mock_setup_database.assert_called_with(use_migrate=False, use_v2_resolver=False)
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above
@patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above
@patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above
@patch(
"litellm.proxy.db.check_migration.check_prisma_schema_diff"
) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above
@patch(
"litellm.proxy.db.prisma_client.should_update_prisma_schema"
) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above
def test_migrations_run_when_the_prisma_cli_is_not_on_path(
self,
mock_should_update_schema,
mock_check_schema_diff,
mock_setup_database,
mock_atexit_register,
tmp_path,
capsys,
):
mock_should_update_schema: MagicMock,
mock_check_schema_diff: MagicMock,
mock_atexit_register: MagicMock,
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
mock_setup_database: MagicMock,
database_startup: proxy_cli._DatabaseStartupDependencies,
) -> None:
from litellm.proxy.proxy_cli import run_server
mock_should_update_schema.return_value = True
empty_bin = tmp_path / "emptybin"
empty_bin: Final = tmp_path / "emptybin"
empty_bin.mkdir()
mock_proxy_module = MagicMock(
mock_proxy_module: Final = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {
k: v
for k, v in os.environ.items()
if k not in ("DATABASE_URL", "DIRECT_URL")
clean_env: Final = {
**{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")},
"DATABASE_URL": "postgresql://test:test@localhost:5432/test",
}
clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test"
clean_env["PATH"] = str(empty_bin)
with (
@ -2058,12 +2066,10 @@ class TestRunServerDbSetup:
"port": 8000,
}
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False)
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup)
assert "prisma CLI is neither on PATH" not in capsys.readouterr().out
mock_setup_database.assert_called_once_with(
use_migrate=True, use_v2_resolver=False
)
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=False)
@pytest.mark.parametrize(
("arguments", "env", "exits"),
@ -2079,21 +2085,21 @@ class TestRunServerDbSetup:
)
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
@patch("litellm.proxy.db.check_migration.check_prisma_schema_diff")
@patch("litellm.proxy.db.prisma_client.should_update_prisma_schema")
def test_startup_fails_when_db_setup_fails(
self,
mock_should_update_schema,
mock_check_schema_diff,
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
arguments,
env,
exits,
capsys,
):
mock_should_update_schema: MagicMock,
mock_check_schema_diff: MagicMock,
mock_atexit_register: MagicMock,
mock_subprocess_run: MagicMock,
arguments: tuple[str, ...],
env: dict[str, str],
exits: bool,
capsys: pytest.CaptureFixture[str],
mock_setup_database: MagicMock,
database_startup: proxy_cli._DatabaseStartupDependencies,
) -> None:
"""A failed PrismaManager.setup_database exits 1 by default; only an explicit
--no-enforce_prisma_migration_check or ENFORCE_PRISMA_MIGRATION_CHECK=false keeps the proxy booting"""
from litellm.proxy.proxy_cli import run_server
@ -2102,20 +2108,22 @@ class TestRunServerDbSetup:
mock_should_update_schema.return_value = True
mock_setup_database.return_value = False
mock_proxy_module = MagicMock(
mock_proxy_module: Final = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {
k: v
for k, v in os.environ.items()
if k not in ("DATABASE_URL", "DIRECT_URL", "ENFORCE_PRISMA_MIGRATION_CHECK")
clean_env: Final = {
**{
k: v
for k, v in os.environ.items()
if k not in ("DATABASE_URL", "DIRECT_URL", "ENFORCE_PRISMA_MIGRATION_CHECK")
},
"DATABASE_URL": "postgresql://test:test@localhost:5432/test",
}
clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test"
outcome = pytest.raises(SystemExit) if exits else nullcontext()
outcome: Final = pytest.raises(SystemExit) if exits else nullcontext()
with (
patch.dict(os.environ, {**clean_env, **env}, clear=True),
@ -2136,27 +2144,27 @@ class TestRunServerDbSetup:
"host": "localhost",
"port": 8000,
}
run_server.main(["--local", "--skip_server_startup", *arguments], standalone_mode=False)
run_server.main(
["--local", "--skip_server_startup", *arguments], standalone_mode=False, obj=database_startup
)
assert (exc_info is not None and exc_info.value.code == 1) is exits
assert ("Database migration failed but continuing startup" in capsys.readouterr().out) is not exits
mock_setup_database.assert_called_once_with(
use_migrate=True, use_v2_resolver=False
)
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=False)
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
@patch("litellm.proxy.db.check_migration.check_prisma_schema_diff")
@patch("litellm.proxy.db.prisma_client.should_update_prisma_schema")
def test_startup_exits_on_non_postgres_database_url(
self,
mock_should_update_schema,
mock_check_schema_diff,
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
):
mock_should_update_schema: MagicMock,
mock_check_schema_diff: MagicMock,
mock_atexit_register: MagicMock,
mock_subprocess_run: MagicMock,
mock_setup_database: MagicMock,
database_startup: proxy_cli._DatabaseStartupDependencies,
) -> None:
"""A sqlite DATABASE_URL must exit immediately, before any prisma call,
instead of stalling on a migration against the postgresql-only schema."""
from litellm.proxy.proxy_cli import run_server
@ -2164,19 +2172,17 @@ class TestRunServerDbSetup:
mock_subprocess_run.return_value = MagicMock(returncode=0)
mock_should_update_schema.return_value = True
mock_proxy_module = MagicMock(
mock_proxy_module: Final = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {
k: v
for k, v in os.environ.items()
if k not in ("DATABASE_URL", "DIRECT_URL")
clean_env: Final = {
**{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")},
"DATABASE_URL": "sqlite:///data/litellm.db",
}
clean_env["DATABASE_URL"] = "sqlite:///data/litellm.db"
with (
patch.dict(os.environ, clean_env, clear=True),
@ -2189,30 +2195,29 @@ class TestRunServerDbSetup:
),
):
with pytest.raises(SystemExit) as exc_info:
run_server.main(
["--local", "--skip_server_startup"], standalone_mode=False
)
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup)
assert exc_info.value.code == 1
mock_setup_database.assert_not_called()
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
@patch("litellm.proxy.db.check_migration.check_prisma_schema_diff")
@patch("litellm.proxy.db.prisma_client.should_update_prisma_schema")
def test_v2_migration_resolver_opts_in_via_env_var(
self,
mock_should_update_schema,
mock_check_schema_diff,
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
):
"""USE_V2_MIGRATION_RESOLVER must select the v2 resolver.
mock_should_update_schema: MagicMock,
mock_check_schema_diff: MagicMock,
mock_atexit_register: MagicMock,
mock_subprocess_run: MagicMock,
capsys: pytest.CaptureFixture[str],
mock_setup_database: MagicMock,
database_startup: proxy_cli._DatabaseStartupDependencies,
) -> None:
"""USE_V2_MIGRATION_RESOLVER=true must select the v2 resolver.
The Helm migrations Job runs `python litellm/proxy/prisma_migration.py`,
which calls run_server with a fixed argv, so a deployment has no way to
pass --use_v2_migration_resolver and an env var is the only route in.
which calls run_server with a fixed argv, so a deployment reaches the
resolver through the env var rather than a CLI flag.
"""
from litellm.proxy.proxy_cli import run_server
@ -2220,20 +2225,18 @@ class TestRunServerDbSetup:
mock_should_update_schema.return_value = True
mock_setup_database.return_value = True
mock_proxy_module = MagicMock(
mock_proxy_module: Final = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {
k: v
for k, v in os.environ.items()
if k not in ("DATABASE_URL", "DIRECT_URL")
clean_env: Final = {
**{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")},
"DATABASE_URL": "postgresql://test:test@localhost:5432/test",
"USE_V2_MIGRATION_RESOLVER": "true",
}
clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test"
clean_env["USE_V2_MIGRATION_RESOLVER"] = "true"
with (
patch.dict(os.environ, clean_env, clear=True),
@ -2245,13 +2248,56 @@ class TestRunServerDbSetup:
},
),
):
run_server.main(
["--local", "--skip_server_startup"], standalone_mode=False
)
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup)
mock_setup_database.assert_called_once_with(
use_migrate=True, use_v2_resolver=True
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True)
@pytest.mark.parametrize(
("skip_server_startup", "migrated", "build_succeeds", "calls_expected", "succeeds"),
(
(True, True, False, ("setup", "build"), True),
(True, True, True, ("setup", "build"), True),
(False, True, False, ("setup", "start"), True),
(False, False, False, ("setup",), False),
(True, False, False, ("setup",), False),
),
ids=(
"failed-build",
"migration-job",
"serving-proxy",
"serving-proxy-whose-migrations-failed",
"failed-migration-job",
),
)
def test_the_migration_job_waits_for_the_index_build_and_a_serving_proxy_starts_it_in_the_background(
self,
skip_server_startup: bool,
migrated: bool,
build_succeeds: bool,
calls_expected: tuple[str, ...],
succeeds: bool,
) -> None:
calls: Final[SimpleQueue[str]] = SimpleQueue()
def setup() -> bool:
calls.put("setup")
return migrated
def build() -> bool:
calls.put("build")
return build_succeeds
def start() -> None:
calls.put("start")
assert (
proxy_cli._setup_database_with_request_log_indexes(
skip_server_startup, setup=setup, build=build, start=start
)
is succeeds
)
assert tuple(calls.get_nowait() for _ in calls_expected) == calls_expected
assert calls.empty()
# --- Module-level helpers for worker startup hook tests ---

View file

@ -9,6 +9,9 @@ an `UPDATE` hidden in the `DO $$ ... $$` block this repo uses for conditional DD
import importlib.util
import sys
from pathlib import Path
from typing import Final
import pytest
_CHECKER_PATH = Path(__file__).resolve().parents[1] / "code_coverage_tests" / "check_migrations_no_data_rewrites.py"
_SPEC = importlib.util.spec_from_file_location("check_migrations_no_data_rewrites", _CHECKER_PATH)
@ -205,6 +208,96 @@ class TestDefaultedColumnsOnRequestLogTables:
assert 'ADD COLUMN ... DEFAULT on "LiteLLM_SpendLogs" rewrites existing rows at boot' in rendered
class TestIndexesOnLogTables:
"""Every CREATE INDEX on a request-log table is rejected: a plain one blocks writes for
the whole build and a concurrent one fails on a partitioned parent, so the migration job
(litellm_proxy_extras/request_log_indexes.py) builds those instead."""
def test_the_original_spend_log_index_statement_is_flagged(self, tmp_path: Path) -> None:
sql: Final = (
"-- CreateIndex\n"
'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" '
'ON "LiteLLM_SpendLogs"("api_key", "startTime");'
)
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_the_original_concurrent_call_id_index_statement_is_flagged(self, tmp_path: Path) -> None:
sql: Final = (
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" '
'ON "LiteLLM_SpendLogs"("litellm_call_id");'
)
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_unique_index_with_if_not_exists_on_error_logs_is_flagged(self, tmp_path: Path) -> None:
sql: Final = 'CREATE UNIQUE INDEX IF NOT EXISTS "ix" ON "LiteLLM_ErrorLogs" ("request_id");'
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_ErrorLogs"',)
def test_a_unique_concurrent_index_on_error_logs_is_flagged(self, tmp_path: Path) -> None:
sql: Final = 'CREATE UNIQUE INDEX CONCURRENTLY "ix" ON "LiteLLM_ErrorLogs" ("request_id");'
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_ErrorLogs"',)
def test_lowercase_schema_qualified_and_only_forms_are_flagged(self, tmp_path: Path) -> None:
sql: Final = (
'create index on "public"."LiteLLM_SpendLogs" ("api_key");\n'
'CREATE INDEX "ix" ON ONLY "LiteLLM_SpendLogs" ("api_key");\n'
'CREATE INDEX CONCURRENTLY "iy" ON "public"."LiteLLM_SpendLogs" ("api_key");'
)
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) * 3
def test_a_comment_between_on_and_the_table_is_flagged(self, tmp_path: Path) -> None:
sql: Final = 'CREATE INDEX CONCURRENTLY "ix" ON /* table */ "LiteLLM_SpendLogs" ("api_key");'
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_a_concurrent_index_with_comments_and_line_breaks_is_flagged(self, tmp_path: Path) -> None:
sql: Final = (
"-- CreateIndex\n"
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "ix"\n'
' ON "LiteLLM_SpendLogs" /* partitioned in some deployments */\n'
' ("api_key", "startTime");\n'
)
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_indexes_on_a_non_log_table_pass_concurrent_or_not(self, tmp_path: Path) -> None:
sql: Final = (
'CREATE INDEX "ix" ON "LiteLLM_VerificationToken" ("token");\n'
'CREATE INDEX CONCURRENTLY "iy" ON "LiteLLM_VerificationToken" ("token");'
)
assert _keywords(tmp_path, sql) == ()
def test_an_index_run_by_execute_is_flagged(self, tmp_path: Path) -> None:
sql: Final = 'DO $$ BEGIN EXECUTE \'CREATE INDEX "ix" ON "LiteLLM_SpendLogs" ("api_key")\'; END $$;'
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_a_marker_does_not_exempt_the_index(self, tmp_path: Path) -> None:
sql: Final = (
'-- data-migration-ok: table is empty at this point\nCREATE INDEX "ix" ON "LiteLLM_SpendLogs" ("api_key");'
)
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_a_marker_on_a_rewrite_still_leaves_the_index_below_it_flagged(self, tmp_path: Path) -> None:
sql: Final = (
"-- data-migration-ok: one row\n"
'UPDATE "LiteLLM_SpendLogs" SET "api_key" = \'k\' WHERE "request_id" = \'r\';\n'
'CREATE INDEX CONCURRENTLY "ix" ON "LiteLLM_SpendLogs" ("api_key");'
)
assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',)
def test_render_points_at_the_migration_job_index_list(self, tmp_path: Path) -> None:
sql: Final = 'CREATE INDEX CONCURRENTLY "ix" ON "public"."LiteLLM_SpendLogs" ("api_key");'
rendered: Final = _scan(tmp_path, sql)[0].render()
assert "20260101000000_fixture/migration.sql:1" in rendered
assert "blocks writes until the build finishes, or fails on a partitioned table" in rendered
assert "REQUEST_LOG_INDEXES in litellm_proxy_extras/request_log_indexes.py" in rendered
@pytest.mark.parametrize(
"name",
("20260823000000_add_spend_logs_api_key_starttime_index", "20260831120001_spend_logs_litellm_call_id_index"),
)
def test_the_inert_index_migrations_scan_clean_without_a_grandfather(self, name: str) -> None:
assert checker.scan_migration(checker.MIGRATIONS_DIR / name) == ()
assert name not in checker.GRANDFATHERED
class TestInsert:
def test_insert_values_is_bounded_and_passes(self, tmp_path):
assert _keywords(tmp_path, "INSERT INTO \"Foo\" (\"id\") VALUES ('a'), ('b');") == ()

254
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-10-20T22:36:52.444783Z"
exclude-newer = "2026-11-07T05:48:32.556148868Z"
exclude-newer-span = "P3D"
[options.exclude-newer-package]
@ -1660,60 +1660,70 @@ wheels = [
[[package]]
name = "ddtrace"
version = "4.11.0"
version = "4.15.6"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "bytecode" },
{ name = "ddtrace-internal" },
{ name = "envier" },
{ name = "opentelemetry-api" },
{ name = "wrapt" },
]
sdist = { url = "https://files.pythonhosted.org/packages/81/51/a628f0177274bab5b67c93a1558fd222babb15286a427e4e8c1d65a265b2/ddtrace-4.11.0.tar.gz", hash = "sha256:260c5b46e80565f4fd08cec2650f707627fb57cd6f8951a2d8c9e6a02b490074", size = 2422158, upload-time = "2026-07-10T08:57:53.396Z" }
sdist = { url = "https://files.pythonhosted.org/packages/f6/94/f0f005257dc49a7062c6fed98a6dbcca7691e9d1db5c64e8a2d14f204cac/ddtrace-4.15.6.tar.gz", hash = "sha256:2914f646d4ae338c644b814c4ad80c963e7aadfb39d1e67b0367a15d5c4bc0db", size = 2655968, upload-time = "2026-10-08T09:46:58.392Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/16/1f/05dc12819ca3a3000028a522cab0025c5dd969e97dc472ac38a7d2f1ea71/ddtrace-4.11.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:0bc55e48da62b6a287b36e01f5876489023178c09b60e8e01608e2e895ccdc4b", size = 7103767, upload-time = "2026-07-10T08:55:10.348Z" },
{ url = "https://files.pythonhosted.org/packages/e8/8a/efcea448bc15b4d95c48c9b476f2ddc5862a53cd4baf58fe50c64763f9d8/ddtrace-4.11.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5df47873c625993e81fabd8f6ae2ab61f66c6805cfd1a8c551a509d5b5d91062", size = 7447104, upload-time = "2026-07-10T08:55:15.794Z" },
{ url = "https://files.pythonhosted.org/packages/39/28/89aa239e47d28900ab394a0b8e0a5549cab631c0e5977016ad9d578eaee7/ddtrace-4.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e857428cc86fe541f5b3768d7d8331e2f1cbde1a2492deedff4619109cbfd5fc", size = 8511839, upload-time = "2026-07-10T08:55:17.609Z" },
{ url = "https://files.pythonhosted.org/packages/30/43/37a5475ade911abae592e121e852922745d842e16294c549cba4ad0f3dbe/ddtrace-4.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc60575f8f47af58e6523f8a802d1f92a4912aa7d4ba71024958421044cba61a", size = 8733381, upload-time = "2026-07-10T08:55:19.66Z" },
{ url = "https://files.pythonhosted.org/packages/10/df/d61f71236a405480c8809522219b184cc486056c32f9cc8bf07cc91e2be9/ddtrace-4.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d65ab60bfb0b2edde6c564bd1f97b99a8ea9bb082c93b265488ba214c606d240", size = 9513521, upload-time = "2026-07-10T08:55:21.921Z" },
{ url = "https://files.pythonhosted.org/packages/79/4f/bff08af986df419179b86fc9c73732836531c2d08ab8dcfe982a6f2e8cb8/ddtrace-4.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7ebfdee1342b5b1cdad8505e040bbfe5bdf6976e820803eaa4a1814417d6bb0e", size = 9788008, upload-time = "2026-07-10T08:55:24.111Z" },
{ url = "https://files.pythonhosted.org/packages/af/b8/2b1fd5b0374edf33ad31adb746d1d1358a44c39a45b1c2a0c4b09651d81e/ddtrace-4.11.0-cp310-cp310-win32.whl", hash = "sha256:6247a431ae3a2622a0e835357a644b1bac81719e28f21748fd7d4034c79c3ed7", size = 5739013, upload-time = "2026-07-10T08:55:27.028Z" },
{ url = "https://files.pythonhosted.org/packages/f2/f5/9bdff59bdce1f80592e70633dc26678c762fff79188c9fb6004c06967708/ddtrace-4.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:9197110982756a97243906ffce7b98e6007c8258a6bbc22451d7b26e8f1b0103", size = 6326099, upload-time = "2026-07-10T08:55:28.962Z" },
{ url = "https://files.pythonhosted.org/packages/76/36/ad0abcc97b63d821ada82cc7452c3070a71f9546f0aa81b50eda1f8ffd41/ddtrace-4.11.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b4074e538e708b0e9d7f812be15db306abbe3a6cb70046f9f786d66daa862344", size = 7105089, upload-time = "2026-07-10T08:55:31.006Z" },
{ url = "https://files.pythonhosted.org/packages/a6/3a/c0f8f70883a70ec9d4050900823ff7b52ad54dc6b2a3d29561727825410f/ddtrace-4.11.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:05f701229233ad8a4ad5d52b8ff2bfd6e2f99601a3bc04367615cd1ce4a01a80", size = 7448237, upload-time = "2026-07-10T08:55:32.962Z" },
{ url = "https://files.pythonhosted.org/packages/4e/46/18a42afab21cd718467e7d7ae3e80898569d9969d731b771b5a8ada10294/ddtrace-4.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c3a557891e91a443e6a3a9b7c31dc96b9963acada0986dab5a3d65507efa16a1", size = 8516433, upload-time = "2026-07-10T08:55:35.011Z" },
{ url = "https://files.pythonhosted.org/packages/da/b2/6367a75518f843872e13c062ac3c329f880b2f4d62ff4de0624bdcf64719/ddtrace-4.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0328bfeb4a6bdc00168a4184aea670eedc073734f6fbc6ca1abfeb6547415d2", size = 8740643, upload-time = "2026-07-10T08:55:37.743Z" },
{ url = "https://files.pythonhosted.org/packages/ae/e8/04f1d7ee4737444d45f81545ce21b01bd9d653a994997ab1e7e04b6ee947/ddtrace-4.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad6f17111746568a58da89b6db5e5d18eb07db7b8ce3be3396631f37391cff50", size = 9518626, upload-time = "2026-07-10T08:55:40.248Z" },
{ url = "https://files.pythonhosted.org/packages/ed/8b/314ba8a81e562a31655328061afb30d26d46ba04aca4dabd7da85c93a9e7/ddtrace-4.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eb30dd327789ec1851bc42194c8db0b93a7ad708c2963d694e3622b684a085f7", size = 9795392, upload-time = "2026-07-10T08:55:42.609Z" },
{ url = "https://files.pythonhosted.org/packages/9e/6e/a580d53c05a771bc53d33a88174e983146fdf879fe5d4f52e51ceb56ee99/ddtrace-4.11.0-cp311-cp311-win32.whl", hash = "sha256:dfe73130778ca22652ae398fc77de3643d50687deb127bf4cd2566b10778125a", size = 5737732, upload-time = "2026-07-10T08:55:45.605Z" },
{ url = "https://files.pythonhosted.org/packages/12/83/094f5b88690f7801129b0d52a7b6a58a95b34ecbfcb337555d323c639faa/ddtrace-4.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:1241f38d52b40b516d620ed68ecd04afbc715f2c1c9f3ddae8a7b86ecf848c91", size = 6329473, upload-time = "2026-07-10T08:55:47.603Z" },
{ url = "https://files.pythonhosted.org/packages/60/72/036d11ceca6fbdc4ef95a1aba210cb3fc72bc912703cba49cf2bc8b610c5/ddtrace-4.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:3013b67e1a172eb0edd8c37559b81760a9db205a671541925ebd0127f496409b", size = 5990873, upload-time = "2026-07-10T08:55:49.705Z" },
{ url = "https://files.pythonhosted.org/packages/f7/e9/e50295e520f7d14ce2b5d900f7a2fe8f096d8f90e9040f8d7f43108b72ce/ddtrace-4.11.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d0eadcf7df3143bf0611f09a351040e475b5df7f512410698cd481a076bca056", size = 7101986, upload-time = "2026-07-10T08:55:56.51Z" },
{ url = "https://files.pythonhosted.org/packages/cb/c6/e8d13546a4b2cd72880d8b0823dd487956d9eb291473d7d03b6e742e5bd9/ddtrace-4.11.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:57eb0616dc5c8ff04b5c560ba6d3def69d2bc618e69ce65cc015f103fc68649e", size = 7455362, upload-time = "2026-07-10T08:55:58.664Z" },
{ url = "https://files.pythonhosted.org/packages/50/c6/2c22d71290df36b87393794d6d803ec8ea96ee54d42286836a7ddf0dbae3/ddtrace-4.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c311109d1e0f3482b7a44126c855eeb796b186f01604ce74c54800aa5892863d", size = 8502274, upload-time = "2026-07-10T08:56:00.869Z" },
{ url = "https://files.pythonhosted.org/packages/19/84/5afef2c9696b289afa1c608e862930c0e447489776a0133da9f2345edd27/ddtrace-4.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:695ec915640072e2917b13d317fe1696086a38924ba02cbe1dcb3918ffb0fa65", size = 8730138, upload-time = "2026-07-10T08:56:03.711Z" },
{ url = "https://files.pythonhosted.org/packages/12/68/a2aff5999fa2b1053b3f50cd78297cbbe7a427f63388fb5f6c500dfb859a/ddtrace-4.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:355a288756a7a5d6b73221d28ba4536cb0271d2ea7909adab3e2ee37d62a569f", size = 9507981, upload-time = "2026-07-10T08:56:06.317Z" },
{ url = "https://files.pythonhosted.org/packages/79/12/53bb54b3451f85dbf3c752c2d60b22f672fd55bb08d7495613a70dc87102/ddtrace-4.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c19888af09f6ad3340f2935ea5c9eb80d9298141e196fd33e544369920aefba", size = 9793730, upload-time = "2026-07-10T08:56:08.893Z" },
{ url = "https://files.pythonhosted.org/packages/9c/7e/d9a52dd0d877ebdce5eb105d9f3d4e3579391ee0106c8335d59aa97006dd/ddtrace-4.11.0-cp312-cp312-win32.whl", hash = "sha256:cd524b299bd15ed14192cb041a35e992003e0d3a9f8831062f9f28bcfacb63e2", size = 5734745, upload-time = "2026-07-10T08:56:11.764Z" },
{ url = "https://files.pythonhosted.org/packages/6d/da/bb7c609ebc368cabb01db38d7022f42a39d30207f484a5e18a306bd9b333/ddtrace-4.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:af663120d3b4116965bebbd79ce8e73a85644816a0917999e8164b6204678cb6", size = 6317182, upload-time = "2026-07-10T08:56:14.156Z" },
{ url = "https://files.pythonhosted.org/packages/30/f3/e3cf49ce0a4c377fdea8203b127984e26ac2c9cad0521fab27d61371d568/ddtrace-4.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:f0b50a1739573318b0a81fb56a8b22ed6ae6dac017b89400cbc76c17b439d9ed", size = 5980198, upload-time = "2026-07-10T08:56:16.458Z" },
{ url = "https://files.pythonhosted.org/packages/d3/f8/89f2b9cb3d413c55a3d1f19a90fda76ec45abdc0ce13a7f6d4aaf10bed1a/ddtrace-4.11.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:16b9f06c841858debb3de2c4a7ce6acbd6967eaa6ce264dbd0e2a58e376b8583", size = 7094539, upload-time = "2026-07-10T08:56:18.943Z" },
{ url = "https://files.pythonhosted.org/packages/ed/98/f262ad401cdd1a8a2394299d12f31bcc037d0e2dc61d3a843f1c855c293f/ddtrace-4.11.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:1cb352d7df16b864f258d4602addf23b8d29d2d416ccc1d0af8d88c614f70a01", size = 7448347, upload-time = "2026-07-10T08:56:21.359Z" },
{ url = "https://files.pythonhosted.org/packages/1f/17/c9123fc263b89934cc1c831b81d90af299d28512740abf008fd963bd006e/ddtrace-4.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:54f4cf656aa452bc37b62fa7939c5bddca751ffd4d315b3c925236d4794767ea", size = 8497865, upload-time = "2026-07-10T08:56:24.143Z" },
{ url = "https://files.pythonhosted.org/packages/39/31/00d529b9ca80c77bf3c6aa7370615e2186fc9ead573ef56cef345e6a4dbf/ddtrace-4.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:776d99ccb01abecf7dbb4910f4a570b6ab625ed47e00bb6c0159d88d4e2ef1c8", size = 8721481, upload-time = "2026-07-10T08:56:26.983Z" },
{ url = "https://files.pythonhosted.org/packages/41/56/759eab73938820054faa6db6eda607753d821b6322ce7679a2b9f728f8c7/ddtrace-4.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5533ce94e50412565a36ce0c8fc6ff6f7490e69b4ecff10c550482e49d5d065e", size = 9505774, upload-time = "2026-07-10T08:56:29.901Z" },
{ url = "https://files.pythonhosted.org/packages/5d/a4/6ab67386244c08340c1c30affffc644b14095e59eb8d2e8d8ee1d4d46d84/ddtrace-4.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:81d147bd3d6fa067f2c72c7e32aba90d5d61ed379850896f1cc7fd1ea3437bb1", size = 9787192, upload-time = "2026-07-10T08:56:32.824Z" },
{ url = "https://files.pythonhosted.org/packages/e3/8d/44e074f927b35007e8bf3dc2b5aeb5ba5e4c97e76d4b8e994aaabddfd648/ddtrace-4.11.0-cp313-cp313-win32.whl", hash = "sha256:bcf1d3ba8c3771b8c75f0b9ecd9038ae0a44c8bb0c01d21458c405a29341518a", size = 5732179, upload-time = "2026-07-10T08:56:35.958Z" },
{ url = "https://files.pythonhosted.org/packages/7e/ec/ff034010a1fd598863a15606aa7867ee6abaf4a5989b1a22e61753229432/ddtrace-4.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:d465b9f6f049fc32278cbfd74e1e593b0c3460aacb622bef0cee71e50ffc4b05", size = 6314643, upload-time = "2026-07-10T08:56:38.565Z" },
{ url = "https://files.pythonhosted.org/packages/bc/dd/ad7da7ca071c76815b16a0523875c0c061902cfd8e9ae6595577464e923e/ddtrace-4.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:00cc0fbfa6b8741477528c3fe7bc70a4a5ea439221f188e14da6907f03ed0125", size = 5976789, upload-time = "2026-07-10T08:56:41.172Z" },
{ url = "https://files.pythonhosted.org/packages/12/1b/37b9a84cc4a95d74d4358a0f78346376bc03625b6a9fa0d568a1587665ca/ddtrace-4.11.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e6f1ca84237e9f52fd71e3458262cfc407dd43337a56e4a6ba9beea88f0958c5", size = 7099136, upload-time = "2026-07-10T08:56:43.766Z" },
{ url = "https://files.pythonhosted.org/packages/f1/42/f227eeb9c5bbb519c67fe11a5326b531f4c1bbd90257e74ab8b72b5ded75/ddtrace-4.11.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:c3ddbf9a5a0cd51b5ab9bca898c85cfce73072054b9f5b577164a95065cb8331", size = 7452151, upload-time = "2026-07-10T08:56:46.822Z" },
{ url = "https://files.pythonhosted.org/packages/eb/c4/35fbc0e8ef585d93f82dadae38246eb3643621f08efe11ca0c20e77e7d42/ddtrace-4.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7c435b708f4956208141c22d941ed60a1551813353f6c1cfc1a5283ec3d80cc5", size = 8507170, upload-time = "2026-07-10T08:56:49.646Z" },
{ url = "https://files.pythonhosted.org/packages/70/99/a9681a53e9b0557497b24a4e0b1495acf2bcb74368e86129e549d6249a4a/ddtrace-4.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:26b3fd7ee97cf7fa1296eecacd393bbbef343cf8bb2d0b1ee39d40cb1d4818fa", size = 8725097, upload-time = "2026-07-10T08:56:53.106Z" },
{ url = "https://files.pythonhosted.org/packages/e0/c6/b18f02120c34abf55e28bc180e872ddbe27614559c01bbcfe5be35b5494a/ddtrace-4.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8d23d66ba9525fd9b9a61c0db8c43e64461066b1d49c75c85f639ea65776367b", size = 9516895, upload-time = "2026-07-10T08:56:56.513Z" },
{ url = "https://files.pythonhosted.org/packages/c9/ed/dc73e8317c229300b0ff8513f903d16ff140eede8084b3b6f76c8646a548/ddtrace-4.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:85e081e2295a77c9b38e5792a097f1df8169e1dd9b3339f820d00183e10e8b1b", size = 9792739, upload-time = "2026-07-10T08:57:00.333Z" },
{ url = "https://files.pythonhosted.org/packages/7c/8e/b97984d9f43e2f36f5317034711140c3bd1fbb5d96880b1c11b9e1e8c3d3/ddtrace-4.11.0-cp314-cp314-win32.whl", hash = "sha256:c267fe4efceb71019f5e6833fdb4fbdff18a51641bdca9fceabbe579984461e4", size = 5831072, upload-time = "2026-07-10T08:57:03.578Z" },
{ url = "https://files.pythonhosted.org/packages/e8/6a/7fb18925588f645d6d777c90d388f9a640adf7b986be7457943717b74170/ddtrace-4.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bebf3cb12ac51c422ad1ed74873746b8398cce845075b234115c2f518a7a652", size = 6455126, upload-time = "2026-07-10T08:57:06.362Z" },
{ url = "https://files.pythonhosted.org/packages/a0/b2/44d7f07ba94c70bd1827669d446a2e979f7efd6e4a409c469a59684d9a7b/ddtrace-4.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:f596cb2da582600e091c8174ab4f8e44bf4f05c50c03cc450e0378c516e0c6e3", size = 6130096, upload-time = "2026-07-10T08:57:09.413Z" },
{ url = "https://files.pythonhosted.org/packages/9b/c8/6b8de8b78503f54a6c19f5b4c9430825976f1a2ef1b8342f21ac47c70e22/ddtrace-4.15.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:68c18fa37cd718f2360c980bbd2928a5c2bf502a173e61a83f25e42321d5b4b8", size = 7998372, upload-time = "2026-10-08T09:44:43.566Z" },
{ url = "https://files.pythonhosted.org/packages/67/30/b5ec13494b638453c568ced825b863c086fc7237528aa30498b4fcb6928a/ddtrace-4.15.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7d7696cf05e1675847303701c0a2f7733014f1b2cb5938ed2038e27fa32a18b2", size = 8382818, upload-time = "2026-10-08T09:44:45.73Z" },
{ url = "https://files.pythonhosted.org/packages/fa/dc/57c5d150061cf27935a2e630ca5334f3fa815d82f0585ba694821f793445/ddtrace-4.15.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:66647b3c9673468e35fe0294d25015ef6f535bb540a9ce63c76c95ff2679e059", size = 9544001, upload-time = "2026-10-08T09:44:47.366Z" },
{ url = "https://files.pythonhosted.org/packages/f5/7b/6651a225df4438761ddd45c21f624d453aca1508d88ed7fd4c880a3a454a/ddtrace-4.15.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4fd260a1339a2df4867f5b0f7ff9111a0f23ba86d6b402e4c3dfe2f71853cd17", size = 9794825, upload-time = "2026-10-08T09:44:49.468Z" },
{ url = "https://files.pythonhosted.org/packages/e4/8d/ffd12a1fa866068c397a1fbbebaebcd6e40f21e618650e95d93860afef44/ddtrace-4.15.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:95a04ff00e401caef808689ae6bd70a9fc92d3572647f9f1bbd43dfacfa053f6", size = 10533745, upload-time = "2026-10-08T09:44:51.57Z" },
{ url = "https://files.pythonhosted.org/packages/bf/19/64e52eced9a66cbb61dd40b110dc25297b1185f1027016876ae366fcaa1e/ddtrace-4.15.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4eb19b57f1cc6ab8e13e57b8257b91f6b0e5bede753320b30e3920a28d4b1a4f", size = 10833139, upload-time = "2026-10-08T09:44:53.716Z" },
{ url = "https://files.pythonhosted.org/packages/9d/ae/0d964c6eb264ec0ab797071db48221acca6990b5837f1a6d804b88b818ec/ddtrace-4.15.6-cp310-cp310-win32.whl", hash = "sha256:740a87a6504617ec7584d5afb2a6169d780571ddc2b44270274aa8291a70a03c", size = 6756230, upload-time = "2026-10-08T09:44:55.961Z" },
{ url = "https://files.pythonhosted.org/packages/0a/2c/35a05709d630b3ad6f5c05dd90d34bf784fcde93e0722f9e1cec111ec5c4/ddtrace-4.15.6-cp310-cp310-win_amd64.whl", hash = "sha256:7aa9f30da1832111de778a17a26bc2bc564f4810d573ac2d8633609c3db48306", size = 7387627, upload-time = "2026-10-08T09:44:57.786Z" },
{ url = "https://files.pythonhosted.org/packages/2d/b5/3ce5d64ed92d87388a83b52ed766866177997585e224d018922b0494a88d/ddtrace-4.15.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b3f6e66e3b303252edce525ca573e7092e741049556ae80a395eadf0685a036f", size = 7997045, upload-time = "2026-10-08T09:44:59.612Z" },
{ url = "https://files.pythonhosted.org/packages/f7/25/b7f15af30cf446fa95bf6835e515c135cc801ddc9ec18ecd665b7a82063a/ddtrace-4.15.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6bec8671c1e0bdadd7fad2513f449c07b9bf4f847bc652aef2720f0e2f74aeaa", size = 8381818, upload-time = "2026-10-08T09:45:01.325Z" },
{ url = "https://files.pythonhosted.org/packages/d9/95/1c597a597123eb99fccd0fc0ef64ee25ad46460e158dbf258136126134c3/ddtrace-4.15.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3017a4d4fe8375ae4555b80419d5259bdc444af485bcca6ee52dcd1a41890d35", size = 9541087, upload-time = "2026-10-08T09:45:03.274Z" },
{ url = "https://files.pythonhosted.org/packages/55/88/7e8c66d3db4739599a8092197940d211a006edca56b8fbbf9bbd34869a27/ddtrace-4.15.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:220fbb49481c6dd85866110477183b70882cb2578020c3bbbb4db2d89d0b41a8", size = 9793899, upload-time = "2026-10-08T09:45:05.384Z" },
{ url = "https://files.pythonhosted.org/packages/84/f8/08c6b74b9fd13df49cfdb95e6a1a86e06c434994db8f4a4a3395f0fee013/ddtrace-4.15.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d50b28793f381bb53e2d7cab5ffeb97c1aa0032f436a1fd5984cb3740d75b10", size = 10534489, upload-time = "2026-10-08T09:45:07.549Z" },
{ url = "https://files.pythonhosted.org/packages/44/6c/03cd4a5cc888f7c540fa9282d3c7c3a7675aae2c622c41668d1a765e0699/ddtrace-4.15.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b5a9edbd149d793b134bdaea28f40db5a199d42135f122c2be325519db55fb75", size = 10834145, upload-time = "2026-10-08T09:45:09.661Z" },
{ url = "https://files.pythonhosted.org/packages/25/6d/ab22cf125e6618597c41b55682dc76617cb6e071ca7381aac346840d6404/ddtrace-4.15.6-cp311-cp311-win32.whl", hash = "sha256:6b6a59fc3dfb081fd7708c9431b94dd17c5148625ec84bc425852cdfbf2f214d", size = 6752572, upload-time = "2026-10-08T09:45:11.785Z" },
{ url = "https://files.pythonhosted.org/packages/b6/38/ffc989894b1d7ea7c06fb249185807c574da5b4572adb9c974fcf02ea65d/ddtrace-4.15.6-cp311-cp311-win_amd64.whl", hash = "sha256:fef3cdf314eee555c5f6b831aeeb826a7eec147e3e1ab85591154a0ad96828f9", size = 7388927, upload-time = "2026-10-08T09:45:14.111Z" },
{ url = "https://files.pythonhosted.org/packages/ab/b8/1b8483fac4a4bcf1d7338d6323b2a3bde576553b111c5a288aee71224613/ddtrace-4.15.6-cp311-cp311-win_arm64.whl", hash = "sha256:4f163cc7fd9d76711c9df54273cbc2bb15933d90d1027fbf6ea3e938f042d23a", size = 7391090, upload-time = "2026-10-08T09:45:16.749Z" },
{ url = "https://files.pythonhosted.org/packages/c1/3f/010c67d812aa0950bb44aeabec69c858e5bd7d31d9fe560ae431e2a08d95/ddtrace-4.15.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c4ca75ae638e105e5c28b7f7b3e1d3762f027ab14a315530e62c71d402b49dc1", size = 7993463, upload-time = "2026-10-08T09:45:18.638Z" },
{ url = "https://files.pythonhosted.org/packages/16/e3/c42f708e8506a75621678e5aac5219b03c8d95014b70e5755287cbe81d2a/ddtrace-4.15.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2f533ad60bb9855ab47a429eeb8b7e28667647c1a482899ac6bd5c00aa525407", size = 8387610, upload-time = "2026-10-08T09:45:20.667Z" },
{ url = "https://files.pythonhosted.org/packages/a3/c5/ea86a9859f8e9b69a448c1ccbb70c51c7989c8c7bf68288356b9f53883a1/ddtrace-4.15.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e00a2943adf9cc886da3cf377b6c32001828565c877a9f0a7905e60eda527dad", size = 9514941, upload-time = "2026-10-08T09:45:22.929Z" },
{ url = "https://files.pythonhosted.org/packages/8d/af/b3bd0fdacb0d2b6baf8bd033ea00e97b06e74e3e504918ecf2f4d176ad6b/ddtrace-4.15.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dfb419eecf9ab3ad774e0fa612f08814530ebd2e1f0aa10128bb0f98eb7cea2e", size = 9772538, upload-time = "2026-10-08T09:45:25.295Z" },
{ url = "https://files.pythonhosted.org/packages/ce/05/f1cb0e9007a5c0d155948726bfefe97fc5fc9d7e886a951a4d51bc1c0236/ddtrace-4.15.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7f8bf477362e29742a6b282fbbfc9c6985c745ba302e4b6a8a53388911ceacae", size = 10511565, upload-time = "2026-10-08T09:45:27.732Z" },
{ url = "https://files.pythonhosted.org/packages/09/90/9f27058af312b213abfb946ab27b334b58b5482aada8b00ce2c41bee2ee6/ddtrace-4.15.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd6c6cd99d6edcffab804755f8246f3c1ecf727bdb241ecfe94f5de31fbb484b", size = 10824106, upload-time = "2026-10-08T09:45:30.202Z" },
{ url = "https://files.pythonhosted.org/packages/8b/9a/f4402bdfb563d103ece19bd56a4622accd0113dfb9ac9b118050757823f9/ddtrace-4.15.6-cp312-cp312-win32.whl", hash = "sha256:903a24868a743e035ff0f7bdc7e397a90c00fac77b551765a9ceae5537c40a44", size = 6756938, upload-time = "2026-10-08T09:45:32.864Z" },
{ url = "https://files.pythonhosted.org/packages/55/fb/0b24d9fbcecfeb72ac2fb24db5b4a4684a0f0ecda73bb2b0c56c14282645/ddtrace-4.15.6-cp312-cp312-win_amd64.whl", hash = "sha256:57cd4ca85c5fef665fc2794c37369e39d6b9f51b1ff37ae8e68ac935360d1561", size = 7375054, upload-time = "2026-10-08T09:45:35.014Z" },
{ url = "https://files.pythonhosted.org/packages/b6/34/09763e279fb865196966cc895fb234cf12350f08fde8a284c796f13d6d34/ddtrace-4.15.6-cp312-cp312-win_arm64.whl", hash = "sha256:be11b8c8fce20b6e02d6069a53d43fc92aec9a50741e0f0dadebd913f2917ae5", size = 7376141, upload-time = "2026-10-08T09:45:37.46Z" },
{ url = "https://files.pythonhosted.org/packages/17/0c/f0b7dfb83917ea3f9c0d072cf8cf9454550ecf26372b19be49b8cf16bbcb/ddtrace-4.15.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8a118c85925d143e88e48e4583500e95a9b88d9c1fabc96df276c086eb515603", size = 7987311, upload-time = "2026-10-08T09:45:40.025Z" },
{ url = "https://files.pythonhosted.org/packages/f5/01/f39c527b10ceef714c16a99d59429ddd8b314c7d5301bff86fab6201dfae/ddtrace-4.15.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:1307ee87674c3d2d93f38bcd258b62b70a41081183e4bc94b03594c855751322", size = 8382130, upload-time = "2026-10-08T09:45:42.316Z" },
{ url = "https://files.pythonhosted.org/packages/e2/29/0c46ee3d6dcce28f8bd82c1dc658e420bed935fc0488a1ffac2853b42b0f/ddtrace-4.15.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fabb658b1ec068ab6046fb1ecaa6e6860bd40338235244c3c156c973c4957c00", size = 9507940, upload-time = "2026-10-08T09:45:44.887Z" },
{ url = "https://files.pythonhosted.org/packages/b7/b0/77d18cb89ff7b2652d734ae00e4869ef703945a0393bc465e00878631099/ddtrace-4.15.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2185df0b730e4bb78dfe40cf61b7e242d4b81724cea4f496ca5a0ae0f5427309", size = 9764498, upload-time = "2026-10-08T09:45:47.528Z" },
{ url = "https://files.pythonhosted.org/packages/64/06/7435d3534b646ddafaf16fced76b0dec8d239492eef1e119d4c3b3fe1259/ddtrace-4.15.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7fc21e13b52290d33bcdec36644eb63eac03f49adb0b36c6d84dc4bb02b9b935", size = 10508643, upload-time = "2026-10-08T09:45:50.63Z" },
{ url = "https://files.pythonhosted.org/packages/34/d0/65db05d5b72fcb88378edd4bf44e88b2dc0c3e8e1c3070a4630e5ec2dfdd/ddtrace-4.15.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:008a5c93db9489861394190b9a9ecffc464a9a83882792c83c005e938fc69df0", size = 10817707, upload-time = "2026-10-08T09:45:53.417Z" },
{ url = "https://files.pythonhosted.org/packages/df/e0/b2ad40242ce8d17ba4432c4f96a7584d93852c442fcb8ac1285cbd10ae96/ddtrace-4.15.6-cp313-cp313-win32.whl", hash = "sha256:d5ca2d38c37194aee97801775193a034f7f27f2610d5075b6a550ee664b044dd", size = 6754044, upload-time = "2026-10-08T09:45:56.603Z" },
{ url = "https://files.pythonhosted.org/packages/1e/e3/7a549bc05e6b604ef3c5ad8fef19a951178d8d93ee7088660d3743e53b7b/ddtrace-4.15.6-cp313-cp313-win_amd64.whl", hash = "sha256:fc35805f2c5b64bd7a29304a5c54932792a47b1436d2550354c051870cd706b4", size = 7370629, upload-time = "2026-10-08T09:45:58.955Z" },
{ url = "https://files.pythonhosted.org/packages/50/dc/a140b6640dd57f143e9137e8e4d04653ba9047c7ef65ff26ca03721c90ff/ddtrace-4.15.6-cp313-cp313-win_arm64.whl", hash = "sha256:03221f056c03728b01a4295ff4a3854db2dc33bcfab3131757f5eb4ca2a5a41b", size = 7378982, upload-time = "2026-10-08T09:46:01.698Z" },
{ url = "https://files.pythonhosted.org/packages/62/de/448c66ce790d6fcd1abbeb08e8e88ef1a5b7fc990f6ec48b818b4d9a8b29/ddtrace-4.15.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:70d5dfc73d970fa1b633e0aad3b9457cb94478958976e139b1c78461a3c48b33", size = 7993200, upload-time = "2026-10-08T09:46:04.284Z" },
{ url = "https://files.pythonhosted.org/packages/2f/87/50bbd2d4fe589aa13038c1fe07defd57e63c1bc83c22b92eac13676b3219/ddtrace-4.15.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:26ead4c2956924d0574b6681d063dee6c8741dca98ed3b747c353f6252eae0d7", size = 8388799, upload-time = "2026-10-08T09:46:07.071Z" },
{ url = "https://files.pythonhosted.org/packages/9e/13/d947c88b0fdd7cddb89174f5e9116aead05a935e1406e2921c820cb7bcab/ddtrace-4.15.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e62ec9c83e9e98122f4544592a6bf7d3338e9ea7ff2e34fe4750ade2b2c3e5", size = 9519810, upload-time = "2026-10-08T09:46:09.975Z" },
{ url = "https://files.pythonhosted.org/packages/48/5a/15711bfd54778a28cca9f7988bcdd03e4d680dfceed7f4db22f6b7a06e73/ddtrace-4.15.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3684507243feb45e7438b68989aae9a7df01b7996112d7fe74846e1d5aeb3726", size = 9771503, upload-time = "2026-10-08T09:46:13.009Z" },
{ url = "https://files.pythonhosted.org/packages/7d/d4/d0e81dff83be9370b90c090828330c4b2f06d74644f331d8f6d9518bad44/ddtrace-4.15.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4882e25808cf6102015d7421d6214aa94a788ce3242fec9a1a726a41dcfd79ac", size = 10521211, upload-time = "2026-10-08T09:46:15.867Z" },
{ url = "https://files.pythonhosted.org/packages/f7/c1/e33b44ebfa4dcb80f66fb2cc99c61d9c7706edd48e5ccb5ad04bcea0d5ad/ddtrace-4.15.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5336630ed8c829333dd4195d817b3f3f8211ddf6074e1c731e884771094fc41c", size = 10826453, upload-time = "2026-10-08T09:46:19.084Z" },
{ url = "https://files.pythonhosted.org/packages/e5/32/2125cf7f03e9e8b68b6a9072e393a3be4af91505ada32bc65811d6965881/ddtrace-4.15.6-cp314-cp314-win32.whl", hash = "sha256:a970bc70c81361c9a27452bfd3335182d722315466109208290a1c3a4c221748", size = 6867770, upload-time = "2026-10-08T09:46:22.66Z" },
{ url = "https://files.pythonhosted.org/packages/b4/a0/c18b1db1cf4843a3254aed03b61c1c9cbd5964bd52c18922f633c7ed2304/ddtrace-4.15.6-cp314-cp314-win_amd64.whl", hash = "sha256:e7e8b948d13737e61a8e0dc1fe082d38431a3d4fb285d23236f1c8f733ee5c67", size = 7538347, upload-time = "2026-10-08T09:46:25.41Z" },
{ url = "https://files.pythonhosted.org/packages/8f/77/6181506d7f7693aceccdc13f5a2dc6b18662ed3eedebc02043f00d502e5e/ddtrace-4.15.6-cp314-cp314-win_arm64.whl", hash = "sha256:11662a7a4e1f51c388450cd1092b57738bb32ef98401c6032754a7e220a5f6d0", size = 7574014, upload-time = "2026-10-08T09:46:28.185Z" },
]
[[package]]
name = "ddtrace-internal"
version = "0.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/0f/51/6dc02752dca5920d6627fb92b7e0611f594a823bbd57ed31167571d819bc/ddtrace_internal-0.1.0.tar.gz", hash = "sha256:7390721276ab5f174cb103088be7d0de22d452f2817c949e8bde1182ff5b961e", size = 2504, upload-time = "2026-07-15T18:35:10.062Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/93/ad/3cae600fdfc41f54653c14339ac1f125381a6fc2a823d5e685b627ff152e/ddtrace_internal-0.1.0-py3-none-any.whl", hash = "sha256:8eeac4a812e003cebafc8be35ef05103ae2b132893dfbcee00a67f4af81404a2", size = 2495, upload-time = "2026-07-15T18:35:09.295Z" },
]
[[package]]
@ -4991,7 +5001,7 @@ source = { editable = "enterprise" }
[[package]]
name = "litellm-proxy-extras"
version = "0.4.100.post1"
version = "0.4.100.post2"
source = { editable = "litellm-proxy-extras" }
[[package]]
@ -5657,75 +5667,77 @@ wheels = [
[[package]]
name = "msgpack"
version = "1.2.1"
version = "1.2.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" }
sdist = { url = "https://files.pythonhosted.org/packages/0a/e7/bb605a7bab2d8425a64b3fa762b39dc1bf1c7e3f11ba6fb5413d6db0ff8c/msgpack-1.2.3.tar.gz", hash = "sha256:32edb81a2b5eb7cd7c9d941b2bfbbb082fd2cd09e0e725930316af6b708db186", size = 196517, upload-time = "2026-09-29T02:33:52.276Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/5b/16/f70100614b69feb3ade7285f08c9c52d6cda0a5c03f3f5e2facd63acb211/msgpack-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c7b398c56ff125feae96c2737abfec5595f1fa0aa186df60c56040b8accb95c", size = 82926, upload-time = "2026-06-18T16:12:31.531Z" },
{ url = "https://files.pythonhosted.org/packages/e4/3c/08ecd5cdfe4e2de43aec79062028ad0f7b2d9b1fea5430068c198ba570da/msgpack-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1548006a91aa93c5da81f3bdcebc1a0d10cea2d25969754fbe848da622b2b895", size = 82730, upload-time = "2026-06-18T16:12:32.894Z" },
{ url = "https://files.pythonhosted.org/packages/19/9f/a70c9cb1a04ecc134005149367dcfe35d167284e8f65035a1e4156ad17b5/msgpack-1.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dabedcd0f23559f3596428c6589c1cd8c6eaed3a0d720795b07b0225d769203", size = 400729, upload-time = "2026-06-18T16:12:34.052Z" },
{ url = "https://files.pythonhosted.org/packages/fa/7f/5ce020168cf0439041526e95aa068c722c016aee21624e331aeabeee2e8e/msgpack-1.2.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83efa1c898e0fc5380fc0cabbf75164c52e3b5cbb45973710d75821928380c73", size = 407625, upload-time = "2026-06-18T16:12:35.239Z" },
{ url = "https://files.pythonhosted.org/packages/79/70/fb7668ce0386819303047057aef6fc1da73b584291d9cff82b821744e2ef/msgpack-1.2.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01e2dd6c9b19d333a00282330cc8a73d38d8dabc306dc5b42cd668c3ac82e833", size = 377891, upload-time = "2026-06-18T16:12:36.684Z" },
{ url = "https://files.pythonhosted.org/packages/3d/dc/9ebe654a73c3aed2e40aa6b52e3c2a02b5f53ef0085fa235a45d5b367f87/msgpack-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:350cb813d0af6e65d2f7ef0d729f7ff5be5a8bce03665892f43e5883d4ecc1b8", size = 391987, upload-time = "2026-06-18T16:12:37.839Z" },
{ url = "https://files.pythonhosted.org/packages/42/eb/b67cf64218a2fa25e1c671fe1d3dbb06cbeb973e71bc4b822da079862d0b/msgpack-1.2.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ee1d9ed27d0497b848923746cf762ed2e7db24f4be7eec8e5cbe8c766aa707b7", size = 374603, upload-time = "2026-06-18T16:12:39.221Z" },
{ url = "https://files.pythonhosted.org/packages/a2/2e/9ee200cde32fd1a0101b4006202fde554c1860adfb9bf7bff31ea4c08df8/msgpack-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:633727297ed063441fd1cda2288865487f33ad14eeb8831afb5f0c396a62cfce", size = 405121, upload-time = "2026-06-18T16:12:40.524Z" },
{ url = "https://files.pythonhosted.org/packages/43/b6/f10117be7ca7a51e8feed699a907b8e663a8cd66e115ae6b4fb30cc7945c/msgpack-1.2.1-cp310-cp310-win32.whl", hash = "sha256:298872ecf9e61950f1c6af4ca969b859ee91783bb920ef6e6172697d0c8aad74", size = 64088, upload-time = "2026-06-18T16:12:41.762Z" },
{ url = "https://files.pythonhosted.org/packages/ba/93/89976c696fb0224662239d952c47b4d1661b34d79a332ef5584facaa8579/msgpack-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:2ff164c1b0bcb740b073b99e945234d0212852fa378e44a208c425379140dbeb", size = 70113, upload-time = "2026-06-18T16:12:42.78Z" },
{ url = "https://files.pythonhosted.org/packages/f4/6b/e9b1cdc042c4458801d2545ed782a95f3d6ba8e270cce8745b8603c7f748/msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22", size = 82812, upload-time = "2026-06-18T16:12:45.022Z" },
{ url = "https://files.pythonhosted.org/packages/0c/3a/dd518a1bf78ed1e9ad8afe57307c079a00eafe4b3068932a27ca1ea56b4f/msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5", size = 82739, upload-time = "2026-06-18T16:12:46.025Z" },
{ url = "https://files.pythonhosted.org/packages/70/e0/7ba9e1542bf0771a27b8b37c1316e3f95ae9d748fd765284655c476ad4ef/msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06", size = 414233, upload-time = "2026-06-18T16:12:47.029Z" },
{ url = "https://files.pythonhosted.org/packages/03/8d/671d81534ea0e2b0e8a121be100020da09eb78861fe3aa8f3ef7dcd3bed1/msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4", size = 423843, upload-time = "2026-06-18T16:12:48.19Z" },
{ url = "https://files.pythonhosted.org/packages/d2/b6/e5c737515ed1f166664b87601b532f58cbb73d8aa6a90b99f7c2c5037e8e/msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8", size = 390772, upload-time = "2026-06-18T16:12:49.624Z" },
{ url = "https://files.pythonhosted.org/packages/a8/46/62ed8c2e87d7021eab19921594d961ef3aa3794eec76c716dc30f3bfd433/msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b", size = 409559, upload-time = "2026-06-18T16:12:50.936Z" },
{ url = "https://files.pythonhosted.org/packages/70/ff/59aa3887b860bbf43532835e192b1c388a17590d6068ae4f8b2bc74c906e/msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e", size = 387838, upload-time = "2026-06-18T16:12:52.161Z" },
{ url = "https://files.pythonhosted.org/packages/09/11/f8563e471093420cf6478cb3271a0175d8402b82d879783d4035d2d03360/msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f", size = 421732, upload-time = "2026-06-18T16:12:53.556Z" },
{ url = "https://files.pythonhosted.org/packages/57/cf/e673683c4c6c90c1022b24c65af4b03eda72b182a1176ef6449069d66acc/msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d", size = 64091, upload-time = "2026-06-18T16:12:54.821Z" },
{ url = "https://files.pythonhosted.org/packages/3f/07/ca212739d179f9083bff2c7c08c24101c3555a334fadc2b876b18768a3ae/msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8", size = 70462, upload-time = "2026-06-18T16:12:55.898Z" },
{ url = "https://files.pythonhosted.org/packages/6d/be/6798347b425e26f35db82e69dd83c09716c856a3714e7bffc4c0860fd830/msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66", size = 65059, upload-time = "2026-06-18T16:12:57.053Z" },
{ url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" },
{ url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" },
{ url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" },
{ url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" },
{ url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" },
{ url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" },
{ url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" },
{ url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" },
{ url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" },
{ url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" },
{ url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" },
{ url = "https://files.pythonhosted.org/packages/b0/ac/dcddcab6f6c20ecb387ca5e980371cdb3f87ff69aeca388be97eebc4c074/msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064", size = 83151, upload-time = "2026-06-18T16:13:12.173Z" },
{ url = "https://files.pythonhosted.org/packages/64/71/fbcfa83a1d6a9c6091942d1cfd070962244664b87427a9a49a6897b1b219/msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056", size = 82351, upload-time = "2026-06-18T16:13:13.194Z" },
{ url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" },
{ url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" },
{ url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" },
{ url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" },
{ url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" },
{ url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" },
{ url = "https://files.pythonhosted.org/packages/ef/c5/133f4512a56e983a93445c836c9d94d88f3bc2e0980ff4b9e577bd8416ce/msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707", size = 64471, upload-time = "2026-06-18T16:13:22.293Z" },
{ url = "https://files.pythonhosted.org/packages/e2/98/577e10b055096a7dd40732358cabaf7180a20c79ed1dcdbb618e4b9deac7/msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9", size = 71274, upload-time = "2026-06-18T16:13:23.455Z" },
{ url = "https://files.pythonhosted.org/packages/ba/ee/0c0048e7cfbef23c6a94791b8959ab28155232e7956de8a305b5ff588f05/msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a", size = 64795, upload-time = "2026-06-18T16:13:24.687Z" },
{ url = "https://files.pythonhosted.org/packages/77/58/cce442852c6b9e1639c7c8ac8fd9143121cb32dab0f308df4d1426a8eb9c/msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d", size = 83610, upload-time = "2026-06-18T16:13:25.724Z" },
{ url = "https://files.pythonhosted.org/packages/60/5c/15b4c7a0182f75ffa90751958ba36a9c01cafee367d49a3edc10ed140b01/msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7", size = 83138, upload-time = "2026-06-18T16:13:26.781Z" },
{ url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" },
{ url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" },
{ url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" },
{ url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" },
{ url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" },
{ url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" },
{ url = "https://files.pythonhosted.org/packages/82/90/8b630fef07d8c5ab457b71ff2c217910c83d333c7a68472c186e87cc504a/msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb", size = 65942, upload-time = "2026-06-18T16:13:36.056Z" },
{ url = "https://files.pythonhosted.org/packages/16/f1/467b81e98b24dd3885d7b1857728797b4ffc76a7a7483af4fb321a07de3c/msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b", size = 72627, upload-time = "2026-06-18T16:13:37.079Z" },
{ url = "https://files.pythonhosted.org/packages/a7/1d/5d8c4c89985feb6acefb82a09e501c60392261856d2408d20bfe4f0360b1/msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7", size = 66908, upload-time = "2026-06-18T16:13:38.23Z" },
{ url = "https://files.pythonhosted.org/packages/1b/02/ad2afb678b4de94496cd432b581759b756a92c1192d8c767edd6b132efdc/msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273", size = 86000, upload-time = "2026-06-18T16:13:39.44Z" },
{ url = "https://files.pythonhosted.org/packages/54/74/0b797484013128837f3b1cbb6cea019277c4de4e377dc512b4d9a0f92940/msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1", size = 86544, upload-time = "2026-06-18T16:13:40.447Z" },
{ url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" },
{ url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" },
{ url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" },
{ url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" },
{ url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" },
{ url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" },
{ url = "https://files.pythonhosted.org/packages/2f/e1/aab6c946570496b78e67804721f3d5e2d62a93081b9b37df77764ef56347/msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1", size = 70914, upload-time = "2026-06-18T16:13:49.385Z" },
{ url = "https://files.pythonhosted.org/packages/13/0a/e608956488a2af014cfe6e3d665e090b8ee42aa14b07f8f95b8880d66b09/msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2", size = 77999, upload-time = "2026-06-18T16:13:50.467Z" },
{ url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" },
{ url = "https://files.pythonhosted.org/packages/6d/aa/5b6b09f835791045282dc5d08431db599a5f4743a69fe2f6670045a2cd85/msgpack-1.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec0030361cc861ac699b2ef1c695b741fa145c88f8667fa3d7e3f73deeb648a3", size = 90927, upload-time = "2026-09-29T02:31:28.286Z" },
{ url = "https://files.pythonhosted.org/packages/c9/91/7b288e9133bd1ba92ca0ca4e7f2a4cfc53cf467d99d8d2f57b9939908fac/msgpack-1.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5c1efdd9181cb1b719ee46865f368a927f1c0c65d577798340b1194545b7515a", size = 89798, upload-time = "2026-09-29T02:31:30.028Z" },
{ url = "https://files.pythonhosted.org/packages/71/9b/5c3dbc450d14645dcec987970692d6ab24008cc33d2155474b1d818486f9/msgpack-1.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c309a7abae1d14ba29a8bd0ddbd704a5e469d8e9bd9c3dee0e4ff53d7ae01d56", size = 450687, upload-time = "2026-09-29T02:31:32.407Z" },
{ url = "https://files.pythonhosted.org/packages/2b/21/ea60a8fd0d9e0897fce823e9fd9bf6742567784b35c7eee8f4a18a56eb19/msgpack-1.2.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bf390259cb25a6a1cd197c65810999b811f64cd38683251538bcc5a1e41f7d3", size = 459808, upload-time = "2026-09-29T02:31:34.282Z" },
{ url = "https://files.pythonhosted.org/packages/ee/f7/42140e6afdac8e94bfedae4cfb67ee004b6ad5c4cadd024df42f759bf3b5/msgpack-1.2.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39b6986c19e1f2dfa549d185dba6ccf1de2e4c0ba10d8cfc0048935b1c5f9109", size = 423845, upload-time = "2026-09-29T02:31:35.713Z" },
{ url = "https://files.pythonhosted.org/packages/19/7b/cd54f27b59dfbdc438a12361fbb6798b66d377a978f946bc9512598290e9/msgpack-1.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fcc6800daac4922960f6eeb7a0dda3dd4105e0bf7bce0e83ebc465a78cb7bdba", size = 445608, upload-time = "2026-09-29T02:31:37.65Z" },
{ url = "https://files.pythonhosted.org/packages/57/38/52bc0dc44cc9f7c2339b632f93d02f8badc78cfb0bb070f2a50a51945e53/msgpack-1.2.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:968583e956d0427878050b371308c5f8647088732ef3e66a117dbe1192ec91e0", size = 421721, upload-time = "2026-09-29T02:31:39.151Z" },
{ url = "https://files.pythonhosted.org/packages/89/e6/451c9a42274fb2be82d8ba8b76a5219c613e20f8de1da521d10cb758a9ef/msgpack-1.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1d6bcec3dbbdb89ca385d3a73e63ceae7b841fa0d7ca7c676f1a7bfe7fb2cdb8", size = 460430, upload-time = "2026-09-29T02:31:40.843Z" },
{ url = "https://files.pythonhosted.org/packages/57/bb/663e3100327b58caaa5fb66379e557a2717dac08bb586f22f885756bee47/msgpack-1.2.3-cp310-cp310-win32.whl", hash = "sha256:a6b63917d60d6df451f328bd6afba8565e33c4afe1f62ec4ad758b78731c827b", size = 67987, upload-time = "2026-09-29T02:31:42.157Z" },
{ url = "https://files.pythonhosted.org/packages/28/7a/a00d5d7abc5601099260e0d0af8fadc54fbfac2191315aa56eaee3641d9d/msgpack-1.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:4c0780095871ecc49a58b2ff6b1b43b25214704da67646557ca287a3f49fb2dd", size = 75572, upload-time = "2026-09-29T02:31:43.544Z" },
{ url = "https://files.pythonhosted.org/packages/2a/95/b9c651ccb9d720b2e2c8d537954dff528ab869a03bf89598145716db823c/msgpack-1.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ec90a9ae3e1169fa1171147340f0e97d941aa19fcd3b34e8339a55933ed042af", size = 90404, upload-time = "2026-09-29T02:31:44.826Z" },
{ url = "https://files.pythonhosted.org/packages/50/cd/fc9e2e367e80f1493e2ec5f610dda558b344eeede296f88976db133e8f2c/msgpack-1.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d7e9cbb0998bbfd363fd9a09c330520d5e9cb323c05b5a1a05865d23ccf2226", size = 89683, upload-time = "2026-09-29T02:31:46.413Z" },
{ url = "https://files.pythonhosted.org/packages/19/9e/1028485c6886c1c117f777cc9b053e541eff0fedb3292dfb1da95040edb5/msgpack-1.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6707d2fa2aa1bb5424ea0b05f44ffc989b15ab41a73ff5855bff4944fec7c8ac", size = 465347, upload-time = "2026-09-29T02:31:47.934Z" },
{ url = "https://files.pythonhosted.org/packages/aa/83/800570e6a22376eb8d599920f70aead4779a63611696f567477c4e85a70f/msgpack-1.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:382b219de3d436de3baba0f4b0c6d4336e8f5858d0eb047918b13b69a71c6c55", size = 477820, upload-time = "2026-09-29T02:31:49.479Z" },
{ url = "https://files.pythonhosted.org/packages/ab/ff/817e4a2052f848d3fb67726908d6e4e7c19f68ee7c19553a82ce7b0ed415/msgpack-1.2.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:186e6c602b8a9968b8e864c67d622a69279f7d1e55ae25f40e3bff7e815b2b62", size = 436656, upload-time = "2026-09-29T02:31:51.18Z" },
{ url = "https://files.pythonhosted.org/packages/3d/42/040cc55dde6a7d92057baac8d1fc9cfb9f4fd4162900e2ec16dc33917a7d/msgpack-1.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9276ba88891338f2617044429dfd080ae008c9868a25f6f1a7d004a35dc9ac0a", size = 460939, upload-time = "2026-09-29T02:31:53.026Z" },
{ url = "https://files.pythonhosted.org/packages/09/93/4dc007bdef930eed247346773bc0189b710078961d3218d5ee7ba59f322c/msgpack-1.2.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c942c21a93f36b3a69e828c8945bb72c94dc2ffe488a2086950c812f3edf046c", size = 433608, upload-time = "2026-09-29T02:31:54.981Z" },
{ url = "https://files.pythonhosted.org/packages/c0/97/a1b944046f283ec89445cb2a982c42233b5b07cc630f9be739f4f1d469a3/msgpack-1.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18a6ed513023001b28dcd3ba54966f6bb90a38274ba8d2640464bcab3a1b81d4", size = 477373, upload-time = "2026-09-29T02:31:56.713Z" },
{ url = "https://files.pythonhosted.org/packages/59/79/ab411d0d172743732ab2503f4c32a22dd1a7d1436a6feecbb160e4b6376a/msgpack-1.2.3-cp311-cp311-win32.whl", hash = "sha256:d0238cd05dec9ffbe0de1071df685ba63e30a36ac155285b1a094e727c38cbe9", size = 67514, upload-time = "2026-09-29T02:31:58.267Z" },
{ url = "https://files.pythonhosted.org/packages/63/8d/6f0cb2b84e484e96278455c26870196d025bb0cec312b226a663f1fa9000/msgpack-1.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:30e1522e4173230dca4d9ad896f038f73c0da6c1edd42f4dbad88ac583cf5d46", size = 75850, upload-time = "2026-09-29T02:31:59.449Z" },
{ url = "https://files.pythonhosted.org/packages/aa/25/f99e13a2c1d3f5a1dcaa5aab27f474e8c4358188bbc68ad79fecb0d1aefe/msgpack-1.2.3-cp311-cp311-win_arm64.whl", hash = "sha256:8ca67f77938ea6a3663aa9bd22b3e031f6da84d665be850abab910ee90728dfd", size = 72338, upload-time = "2026-09-29T02:32:00.885Z" },
{ url = "https://files.pythonhosted.org/packages/af/12/4d7c6d6203416d9fbf0f59ebaa805e70fb929b93a41b611bc821ec5964a0/msgpack-1.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:89c930aece4e972b208ba589c8410b4167b05e411a5ea2cb25fd96f8bc47ee43", size = 91577, upload-time = "2026-09-29T02:32:02.141Z" },
{ url = "https://files.pythonhosted.org/packages/eb/c7/8576ad39f4ca42ddad26f68eb8621d2d0a60501193d480f504bd9d7f36c4/msgpack-1.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:905a189853d6bdb204c7ae5f4ab77fb857448abfff574d3d93c62e2815b24b4f", size = 90027, upload-time = "2026-09-29T02:32:03.508Z" },
{ url = "https://files.pythonhosted.org/packages/0a/3a/aa9c580aea1314529a0f3562461479780b0d254b064f0880956bfbcc74a8/msgpack-1.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3d7b3d0018746b5997dd6b14a1870b07cc4c327d9101145d94a1fc264a51a06", size = 460343, upload-time = "2026-09-29T02:32:04.906Z" },
{ url = "https://files.pythonhosted.org/packages/3a/cf/9c2e4d6c179529d5bf4a64cff76fa581486569e9fbdd35bd98f51cb624bf/msgpack-1.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede33b2892ceb976283e009ad12fa1834cfdf1f9c43ee9c97849fc588d00a618", size = 472998, upload-time = "2026-09-29T02:32:06.69Z" },
{ url = "https://files.pythonhosted.org/packages/7b/41/915c81fe6df2d3cbdb0dece4f1a5cd313e1cd2abd9f501d0f50c0582517e/msgpack-1.2.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:666ef5601ab0e6e345e47febc96aa81143cc932201543480cbb9499164f05ffb", size = 423216, upload-time = "2026-09-29T02:32:08.739Z" },
{ url = "https://files.pythonhosted.org/packages/a2/e7/7dda8b1039abfd9bba4c5068172c67135c9e33089f503512db9226f23c24/msgpack-1.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87cf2ef05ff2f2493ba29fcdaef27e960ca64dacfd13460ae29e6f92e0ed05bb", size = 451218, upload-time = "2026-09-29T02:32:10.517Z" },
{ url = "https://files.pythonhosted.org/packages/16/5b/ce995c1ed4a0522b7f2d034bc2034fd63005f240b945961b70fb56fbaf3d/msgpack-1.2.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b774ff994d844e541439ac5d2d49a14def4104830c3465e9394c153f86200ffb", size = 422453, upload-time = "2026-09-29T02:32:11.956Z" },
{ url = "https://files.pythonhosted.org/packages/d2/3f/ce191fb87e2650d0166b34c437e499ee4a7f9db9c1eb164f41725eb6160e/msgpack-1.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaf7e82249837e3aa97297b34a0bb9ff562027381631e057cea6e1367f10b438", size = 469003, upload-time = "2026-09-29T02:32:13.663Z" },
{ url = "https://files.pythonhosted.org/packages/42/35/539123407fe200fb16609c835675496fbeb6017ace9fc93909f0613223ae/msgpack-1.2.3-cp312-cp312-win32.whl", hash = "sha256:7c047250096f9fc19dba26e3d1639b5e7a84114003605c94def667149a70ced1", size = 68303, upload-time = "2026-09-29T02:32:15.02Z" },
{ url = "https://files.pythonhosted.org/packages/6f/4c/331b45f9b86fbda6b9e103244d189068e51f726d8c40021ed66e1f2c415e/msgpack-1.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:3ec409b0d6aa8e9eec6eaf881b893caa215dbe68c5319ca96e8a271d81bb111d", size = 76744, upload-time = "2026-09-29T02:32:16.344Z" },
{ url = "https://files.pythonhosted.org/packages/13/9f/fb572dc42b9fac06c7ea848aaee6e140d84469743bd1402bc07089fc4566/msgpack-1.2.3-cp312-cp312-win_arm64.whl", hash = "sha256:59612b4ed48a04cf024584218e813562f3b30a3bafa5f55abe300b15da314751", size = 71580, upload-time = "2026-09-29T02:32:17.617Z" },
{ url = "https://files.pythonhosted.org/packages/1f/8b/3824d65e912e925d09ce30d9130fa9970d6d2855d7888b13639a6604967f/msgpack-1.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:21bfa4d2aa0b04c1806ef778a1199e9e53ea2441bcbf284420a32083896320b8", size = 91728, upload-time = "2026-09-29T02:32:18.949Z" },
{ url = "https://files.pythonhosted.org/packages/05/e6/df7f2c9ebb94760113debbcea2bd3afe5fdab88a4f7bec1b618755517460/msgpack-1.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:db84203b13aecc222f465061397fdd5b53b7ae73d2c95ffc1c8dc5be0153a709", size = 89955, upload-time = "2026-09-29T02:32:20.224Z" },
{ url = "https://files.pythonhosted.org/packages/08/6a/e5fc57136e8bacccb2b39627dea2cd546540a06181e22fe6db90e15b3ae4/msgpack-1.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e0d7950ca3c1bbae291d0552dd3bb2792fc680629c4c0d44e47e5bab969f3ca", size = 454930, upload-time = "2026-09-29T02:32:21.771Z" },
{ url = "https://files.pythonhosted.org/packages/b0/30/c394d37898db9212d1693456cdf363c7e1a097d0b63e10664007f3df3ec1/msgpack-1.2.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:07c9733089d1b176c3dd2f7fa268452f9d5d784d076473499d754a58e8d1fbbb", size = 466866, upload-time = "2026-09-29T02:32:23.742Z" },
{ url = "https://files.pythonhosted.org/packages/4a/c8/1e4ddf6f6b829b3ee6c530c79dfae89cb609d2b0eedb5e0ae716851c52d1/msgpack-1.2.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f24a43b3560e20f825b807fe1e874bd73d53abaf8bbdcf258a6eb152cddbc1f5", size = 418715, upload-time = "2026-09-29T02:32:25.262Z" },
{ url = "https://files.pythonhosted.org/packages/11/a5/f460ba6d7a12d4301002f3efbb8f841e8bdc9c5fc98d771689677a352885/msgpack-1.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6576f348ed6cc4f31db6fd915a8e94245f042f50eae08d48732425e70638ea37", size = 446489, upload-time = "2026-09-29T02:32:26.988Z" },
{ url = "https://files.pythonhosted.org/packages/49/23/adface88db909bed321c85dd673655152d4a514c67e1f0800eb51c777d07/msgpack-1.2.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cd5a9f9f86a52c24713679aa2631956835f3842512964ff93f736ff76f1f530d", size = 416998, upload-time = "2026-09-29T02:32:28.606Z" },
{ url = "https://files.pythonhosted.org/packages/36/00/5bb3a239ccfc3763c4d0fa49b13b1b7010b00182c499ab3c1fecfe6294bc/msgpack-1.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f9ddd28d3e9bbc602a9dced1591882c7fb9ab776eef8837da2c326fde19e2853", size = 463288, upload-time = "2026-09-29T02:32:30.375Z" },
{ url = "https://files.pythonhosted.org/packages/29/8c/456df77f00d701df9d6980ffb80291bce6e4e2e112e25a4dfae216f0715a/msgpack-1.2.3-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:62cc1a4ef0e553bac32c8342e1f04834aca7de276b92744eb7307db77759b890", size = 53347, upload-time = "2026-09-29T02:32:31.867Z" },
{ url = "https://files.pythonhosted.org/packages/9d/22/ce780be666f89b77cdb855daa9ec62e87bb7f69e9f403e4a5d83a2b2208f/msgpack-1.2.3-cp313-cp313-win32.whl", hash = "sha256:d2f9c4f85e47a44d26d5baf3b041eef23436e224d44eed273f01bd8a12048d9f", size = 68258, upload-time = "2026-09-29T02:32:33.163Z" },
{ url = "https://files.pythonhosted.org/packages/51/06/c3def9bc4db283103c5901b302ee2a4305cb1e69729244f94d9bd8f8e8e7/msgpack-1.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:bb89b5dc30469c84bbf8684826eb851d82412ca95690e111b9ac5e8fb343961a", size = 76569, upload-time = "2026-09-29T02:32:34.412Z" },
{ url = "https://files.pythonhosted.org/packages/12/9f/cef344073858b80adb92d6ea342e20b0eae7a8f6fe70281b69cf03707270/msgpack-1.2.3-cp313-cp313-win_arm64.whl", hash = "sha256:471e12a6a42498a31490c206e0069e343b6a7c35db540be73a879eb06f5be047", size = 71530, upload-time = "2026-09-29T02:32:35.892Z" },
{ url = "https://files.pythonhosted.org/packages/3f/8e/f777f74e38731c428857933c8011596f2d2f3160c821152f23b6ffba862f/msgpack-1.2.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3a31905206722103a84c1f72633fe30692cff6732c9d262e09a27dbc468797c8", size = 92042, upload-time = "2026-09-29T02:32:37.464Z" },
{ url = "https://files.pythonhosted.org/packages/a0/71/551608543ee5d590f7e8d522267665d6d9946866ad2a2a70a770f7c70793/msgpack-1.2.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3372475211a9ce1a23acefe512cb3e121d18c95dc74ed56cb1819ef40836ebf4", size = 90578, upload-time = "2026-09-29T02:32:38.883Z" },
{ url = "https://files.pythonhosted.org/packages/ea/11/6d78ce5a9a58bf9ba7b1b6a8f649173b030e6770c8019cf330b91825ee5d/msgpack-1.2.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9324c54995641c3d1f92a9d55093c8cde0ffa2fbc87a467a688ef60428393220", size = 454352, upload-time = "2026-09-29T02:32:40.34Z" },
{ url = "https://files.pythonhosted.org/packages/3d/08/feb9a196269ba7809f44f9117d9e4a601c41c313f6144fd0c337293a5488/msgpack-1.2.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8ef3a66e4b52d2d7fdd90df2984670124b2ff7546d76bb25dcf68ef47f7df58", size = 462562, upload-time = "2026-09-29T02:32:42.176Z" },
{ url = "https://files.pythonhosted.org/packages/f5/77/3a674f366def24140b103d1ffd4fd27b3d912a13e47da67422afa16bebb3/msgpack-1.2.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:902f3490db0e07a7d40b48536a85c9b28fbf1397e7e1658a45a55f958e303620", size = 418134, upload-time = "2026-09-29T02:32:43.693Z" },
{ url = "https://files.pythonhosted.org/packages/48/82/944e71f280577490d99a3951cbce21aa4cbe04e7ab42cb373fd668af883c/msgpack-1.2.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e51eca14fbb65c4e0a5a9657346962bd3dca78c08e04e3d4dee70ef48687d30", size = 445937, upload-time = "2026-09-29T02:32:45.739Z" },
{ url = "https://files.pythonhosted.org/packages/b1/ec/feddd629c4a3edf1395313680450c525086cceab56dec0d4de9da9ccb618/msgpack-1.2.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f42f146752eedb6765f07dcc04d72dab0a25779ec8d4a88c0085263ce114f22c", size = 416450, upload-time = "2026-09-29T02:32:47.558Z" },
{ url = "https://files.pythonhosted.org/packages/e4/59/263a10f8c4613ba0713f48cbda7695ac8dd6d6fab2fcbc9168f03f23a94d/msgpack-1.2.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0ed5823c4efc20fe87d3530665f40ec18a002be003114814c21235cc8d256207", size = 459546, upload-time = "2026-09-29T02:32:49.145Z" },
{ url = "https://files.pythonhosted.org/packages/1e/21/addcfa1e583cfc8a22fbdc57526621b5decd7ad676ae12e9150b7be1be5d/msgpack-1.2.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:2487453ca1b6104442c6442f9a1a8fee1fe8f428a70d99d4cba799108b304150", size = 53462, upload-time = "2026-09-29T02:32:50.708Z" },
{ url = "https://files.pythonhosted.org/packages/8d/2c/3cb5c8524a1335ee27ca952c7ab78d375a16fea8e18ae3767ba0c880416c/msgpack-1.2.3-cp314-cp314-win32.whl", hash = "sha256:6df430419f2338cb71e4a34d6e64f83c88ccd321f91f40ba4513400b36d864ec", size = 70294, upload-time = "2026-09-29T02:32:52.037Z" },
{ url = "https://files.pythonhosted.org/packages/23/f9/9172ff3cdb85d160ad06df5e2708a5fce7682982a5eee8d31869b9f69d2e/msgpack-1.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:84a6616d396ec1bc18a1e83e67c96a393ec35dfe5e17434a5be7b9aa0fe988ab", size = 77778, upload-time = "2026-09-29T02:32:53.429Z" },
{ url = "https://files.pythonhosted.org/packages/04/e8/b4c23178bcf605ae17cec48a75530dd69d49b0a5a6f5f4df5c47d59f746e/msgpack-1.2.3-cp314-cp314-win_arm64.whl", hash = "sha256:7a003b02c6ee2eea6dfe0bb08818631e3597e69f0131f2a8250488a1cc553290", size = 73794, upload-time = "2026-09-29T02:32:54.763Z" },
{ url = "https://files.pythonhosted.org/packages/66/b1/92704be352c4f428b7e0a0e0fb210cb1aa2b1c42c102b8dc22d34b82fac0/msgpack-1.2.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ccea05b5542f6d283fef3f0a8e93a7f0be90af0ddeeef84c25c0216ba76dcae1", size = 93721, upload-time = "2026-09-29T02:32:56.342Z" },
{ url = "https://files.pythonhosted.org/packages/49/78/9c91f1e86cadcbc100b3780fd429c3715648704032a612e77a00646ebe79/msgpack-1.2.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b1631e12fe572e181cd77e831f69335d6cd5278eac22e3db3f33cf264ac2ac18", size = 94256, upload-time = "2026-09-29T02:32:58.056Z" },
{ url = "https://files.pythonhosted.org/packages/91/4d/270f9725921ae88a29d37a774a77ac24f0ef1411fc960a63f5a4665e81b4/msgpack-1.2.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e54394b7dbe2e12ab032d9d21feef7bb61a90a150a2623633ba3781ba69dcb1f", size = 471673, upload-time = "2026-09-29T02:32:59.886Z" },
{ url = "https://files.pythonhosted.org/packages/48/b8/eaa8d930f72dc1d1dd79511dc2ccf965922b059f2f0ed3b30aebac8c4b11/msgpack-1.2.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63bb7448a1e9111319ae2430c09a5596140c160422830d6271bc75730ff2ff9a", size = 466257, upload-time = "2026-09-29T02:33:01.517Z" },
{ url = "https://files.pythonhosted.org/packages/5b/5a/97adc805037bc7e24c4e2f711bbcd3b28be8ec9aea3e778f18208cfbdb46/msgpack-1.2.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:382bc88fe90f29f5ac8a0b65c7046ff255356f2f2f3186c30e370215736fa1dc", size = 418484, upload-time = "2026-09-29T02:33:03.402Z" },
{ url = "https://files.pythonhosted.org/packages/0d/7e/1c53302606fe436ab48ba539ebafafe4a6a9efe12c4f04dc7eb36912d93e/msgpack-1.2.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c77e27790ad72989db783d5303825fba0b71550f00a490efba35cde7dc4b719f", size = 454064, upload-time = "2026-09-29T02:33:04.977Z" },
{ url = "https://files.pythonhosted.org/packages/00/2d/9ee0170f638907b396c15c6cd26b3e54f869159efc6206683acfd8f696e1/msgpack-1.2.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:700bc0fc9e968a292b9137ee70e7a012f7e115bf0107ce45e3a88202788dfc1e", size = 417901, upload-time = "2026-09-29T02:33:06.489Z" },
{ url = "https://files.pythonhosted.org/packages/cc/d2/905c84490a75cd15a27065407cd085d201f7d392e1e0411f49f03fd31ade/msgpack-1.2.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bd5f91ea75c45cafcc5433ba8fae59b708b736ec178d2441c40c499e9e079db", size = 459896, upload-time = "2026-09-29T02:33:08.361Z" },
{ url = "https://files.pythonhosted.org/packages/37/cd/4ce5809b9ab3b114d7cca64863e436820fa1614b49d55ccb93d49824ac2d/msgpack-1.2.3-cp314-cp314t-win32.whl", hash = "sha256:7995a7c6a62a1d6e7df211b4a16de513bd99fd053525050a319f80f44fb8015e", size = 75983, upload-time = "2026-09-29T02:33:10.023Z" },
{ url = "https://files.pythonhosted.org/packages/8a/31/853bb580744c24be0dbd8b090c3e6987dce466a1fc840fe50c0ac2ef9044/msgpack-1.2.3-cp314-cp314t-win_amd64.whl", hash = "sha256:bfe7d5b62cbe7aa664f0b3e2c49077f10fcdd06183d3014f8271ff3c5edbfbf9", size = 83757, upload-time = "2026-09-29T02:33:11.441Z" },
{ url = "https://files.pythonhosted.org/packages/0d/49/9f1b2ee484414eef9e21ee2b2b23b482bb71433ab9bac1da03cbda15ebf5/msgpack-1.2.3-cp314-cp314t-win_arm64.whl", hash = "sha256:1f585407f740a9eac04a3bb82c61d68a0ea78f90e29e670bfb086b9ce3a518dd", size = 78128, upload-time = "2026-09-29T02:33:13.063Z" },
]
[[package]]
@ -6108,11 +6120,11 @@ wheels = [
[[package]]
name = "oauthlib"
version = "4.0.0"
version = "4.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/7a/d8/a1bcc8ba112a627f8ffbdc212a78ce18d3ac07e91a5ca65d27918eee25a1/oauthlib-4.0.0.tar.gz", hash = "sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86", size = 187232, upload-time = "2026-09-28T06:01:18.77Z" }
sdist = { url = "https://files.pythonhosted.org/packages/33/17/2bb072de0b990303c985f26869fd47bb179024a051f47e7f9d208eb92f8e/oauthlib-4.0.1.tar.gz", hash = "sha256:3d73febdb6faa00f277c81fb04be7b7586fb96fc3b7f0b9c772dada69a25ede7", size = 191219, upload-time = "2026-10-31T09:52:58.008Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" },
{ url = "https://files.pythonhosted.org/packages/df/ef/3b8adde1b32a6f540758bff734d3442c621734c83708e5f02b0ba2f43b84/oauthlib-4.0.1-py3-none-any.whl", hash = "sha256:498a298876e996301950a5b053d12c31628bb7b895db03666430c69ce182eb61", size = 161934, upload-time = "2026-10-31T09:52:56.206Z" },
]
[[package]]
@ -7770,11 +7782,11 @@ wheels = [
[[package]]
name = "pyasn1"
version = "0.6.4"
version = "0.6.5"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" }
sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" },
{ url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" },
]
[[package]]