diff --git a/db_scripts/partition_spend_logs.sql b/db_scripts/partition_spend_logs.sql index c153a67eaec..750c1a934aa 100644 --- a/db_scripts/partition_spend_logs.sql +++ b/db_scripts/partition_spend_logs.sql @@ -55,6 +55,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx" RENAME TO "LiteLLM_SpendLogs_legacy_session_id_idx"; ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" RENAME TO "LiteLLM_SpendLogs_legacy_api_key_startTime_idx"; +ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" + RENAME TO "LiteLLM_SpendLogs_legacy_litellm_call_id_idx"; CREATE TABLE "LiteLLM_SpendLogs" ( LIKE "LiteLLM_SpendLogs_legacy" INCLUDING DEFAULTS INCLUDING GENERATED diff --git a/litellm-proxy-extras/litellm_proxy_extras/_redaction.py b/litellm-proxy-extras/litellm_proxy_extras/_redaction.py new file mode 100644 index 00000000000..152ae29b43b --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/_redaction.py @@ -0,0 +1,50 @@ +import functools +import os +import re +from collections.abc import Callable +from typing import Final +from urllib.parse import unquote, urlsplit + +_REDACTED: Final = "REDACTED" +_PASSWORD_QUERY_KEYS: Final = frozenset(("password", "sslpassword")) + + +@functools.cache +def _secret_shape_redactor() -> Callable[[str], str]: + try: + from litellm._logging import redact_secrets + except ImportError: + return lambda text: text + return redact_secrets + + +def _url_passwords(url: str) -> frozenset[str]: + try: + parts: Final = urlsplit(url) + except ValueError: + return frozenset() + query_pairs: Final = tuple(pair.partition("=") for pair in parts.query.split("&")) + raw_query_passwords: Final = tuple( + value for key, separator, value in query_pairs if separator and key.lower() in _PASSWORD_QUERY_KEYS + ) + raw_passwords: Final = ((parts.password,) if parts.password else ()) + raw_query_passwords + return frozenset(password for password in raw_passwords + tuple(map(unquote, raw_passwords)) if password) + + +def _configured_database_passwords() -> frozenset[str]: + database_url: Final = os.getenv("DATABASE_URL") + direct_url: Final = os.getenv("DIRECT_URL") + database_passwords: Final = _url_passwords(database_url) if database_url else frozenset() + direct_passwords: Final = _url_passwords(direct_url) if direct_url else frozenset() + return database_passwords | direct_passwords + + +def _redact_credentials(text: str) -> str: + """Mask configured database passwords before passing the text to LiteLLM redaction.""" + passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True) + alternation: Final = "|".join(re.escape(password) for password in passwords) + password_pattern: Final = ( + re.compile(rf"(? Mapping[str, str]: + database_url: Final = environment.get("DATABASE_URL") + direct_url: Final = environment.get("DIRECT_URL") + if not database_url or not direct_url: + return environment + schema: Final = next((value for key, value in parse_qsl(urlsplit(database_url).query) if key == "schema"), "public") + direct: Final = urlsplit(direct_url) + parameters: Final = tuple((key, value) for key, value in parse_qsl(direct.query) if key != "schema") + return { + **environment, + "DATABASE_URL": urlunsplit(direct._replace(query=urlencode((*parameters, ("schema", schema))))), + } + + +@dataclass(frozen=True, slots=True) +class _LockResult: + acquired: bool + + +def _try_lock(connection: "psycopg.Connection[tuple[object, ...]]", key: int = MIGRATION_LOCK_KEY) -> bool: + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(_LockResult)) as cursor: + row: Final = cursor.execute("SELECT pg_try_advisory_xact_lock(%s) AS acquired", (key,)).fetchone() + return row is not None and row.acquired + + +def _coordination_failure(database_url: str, exc: "psycopg.Error", *, connecting: bool) -> str: + import psycopg + + prefix: Final = "Lost or could not establish v2 migration coordination with the database: " + raw: Final = str(exc) + unquoted: Final = re.sub(r"\"[^\"]*\"|'[^']*'", "", raw) + detail: Final = _redact_credentials(raw).strip() + if ( + not connecting + or not isinstance(exc, psycopg.OperationalError) + or any(marker in unquoted for marker in _SERVER_ANSWERED_MARKERS) + ): + return prefix + detail + parsed_url: Final = urlsplit(database_url) + try: + port: Final = parsed_url.port + except ValueError: + return prefix + f"P1001: Can't reach database server. {detail}" + host: Final = parsed_url.hostname or "localhost" + return prefix + f"P1001: Can't reach database server at `{host}:{port or 5432}`. {detail}" + + +@dataclass(frozen=True, slots=True) +class MigrationCoordinator: + connection: "psycopg.Connection[tuple[object, ...]]" + + def check_connection(self) -> None: + self.connection.execute("SELECT 1") + + def acquire_prisma_lock(self) -> None: + deadline: Final = time.monotonic() + migration_lock_timeout() + while time.monotonic() < deadline: + if _try_lock(self.connection, 72707369): + return + time.sleep(min(random.uniform(0.5, 1.5), max(0.0, deadline - time.monotonic()))) + raise RuntimeError( + "Timed out waiting for Prisma's lock to recover migration history. LiteLLM startup has stopped. " + "Another migration or a pooled database session may still hold the lock. Check the database lock holder. " + "When using a transaction pooler, configure DIRECT_URL to reach the same database without the pooler." + ) + + +@contextmanager +def migration_lock(database_url: str) -> Generator[MigrationCoordinator, None, None]: + import psycopg + + wait_seconds: Final = migration_lock_timeout() + deadline: Final = time.monotonic() + wait_seconds + try: + connection: Final = psycopg.connect(database_url, connect_timeout=10, autocommit=True) + except psycopg.Error as exc: + raise RuntimeError(_coordination_failure(database_url, exc, connecting=True)) from None + try: + with connection: + coordinator: Final = MigrationCoordinator(connection) + logger.info("Waiting for the v2 migration coordinator lock (up to %ss)", wait_seconds) + while time.monotonic() < deadline: + with connection.transaction(): + if _try_lock(connection): + logger.info("Acquired the v2 migration coordinator lock") + + yield coordinator + coordinator.check_connection() + return + time.sleep(min(random.uniform(0.5, 1.5), max(0.0, deadline - time.monotonic()))) + except psycopg.Error as exc: + raise RuntimeError(_coordination_failure(database_url, exc, connecting=False)) from None + raise RuntimeError( + f"Timed out waiting for another v2 migration resolver after {wait_seconds}s. " + f"Check the running migration or increase {MIGRATION_LOCK_TIMEOUT_ENV_VAR}." + ) + + +@contextmanager +def held_migration_lock(connection: "psycopg.Connection[tuple[object, ...]]") -> Generator[bool, None, None]: + """A session-level, non-blocking hold of the migration coordinator lock on an autocommit + connection, for DDL that cannot run inside a transaction (`CREATE INDEX CONCURRENTLY`). + Yields whether the lock was acquired; a v2 resolver or another migration job's index build + holding it yields False. Released on exit.""" + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(_LockResult)) as cursor: + row: Final = cursor.execute("SELECT pg_try_advisory_lock(%s) AS acquired", (MIGRATION_LOCK_KEY,)).fetchone() + acquired: Final = row is not None and row.acquired + try: + yield acquired + finally: + if acquired: + connection.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_KEY,)) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py b/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py new file mode 100644 index 00000000000..5a55b35b255 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py @@ -0,0 +1,204 @@ +import hashlib +import re +import subprocess +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Final +from uuid import uuid4 + +from litellm_proxy_extras import prisma_toolchain +from litellm_proxy_extras._logging import logger +from litellm_proxy_extras.migration_lock import MigrationCoordinator + +if TYPE_CHECKING: + import psycopg + + +@dataclass(frozen=True, slots=True) +class MigrationProgress: + checksum: str + applied_steps_count: int + logs: str + id: str = "" + finished: bool = False + + def confirms_completion(self, script: bytes) -> bool: + return ( + self.applied_steps_count == 1 + and not self.logs.strip() + and self.checksum == hashlib.sha256(script).hexdigest() + ) + + +def _migration_records( + connection: "psycopg.Connection[tuple[object, ...]]", schema: str, migration: Path +) -> tuple[MigrationProgress, ...]: + from psycopg import sql + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(MigrationProgress)) as cursor: + records: Final = cursor.execute( + sql.SQL( + "SELECT id, checksum, applied_steps_count, coalesce(logs, '') AS logs, " + "finished_at IS NOT NULL AS finished FROM {} " + "WHERE migration_name = %s AND rolled_back_at IS NULL" + ).format(sql.Identifier(schema, "_prisma_migrations")), + (migration.parent.name,), + ).fetchall() + return tuple(records) + + +def recover_completed_migration(coordinator: MigrationCoordinator, schema: str, migration: Path) -> bool: + """Finish a proven successful row without erasing its durable completion evidence. + + The caller commits this checkpoint before running another Prisma command. + """ + from psycopg import sql + + coordinator.acquire_prisma_lock() + records: Final = _migration_records(coordinator.connection, schema, migration) + unfinished: Final = tuple(record for record in records if not record.finished) + script: Final = migration.read_bytes() + if not unfinished: + return any(record.checksum == hashlib.sha256(script).hexdigest() for record in records) + if len(unfinished) != 1 or not unfinished[0].confirms_completion(script): + return False + progress: Final = unfinished[0] + result: Final = coordinator.connection.execute( + sql.SQL( + "UPDATE {} SET finished_at = current_timestamp " + "WHERE id = %s AND checksum = %s AND applied_steps_count = 1 " + "AND finished_at IS NULL AND rolled_back_at IS NULL AND coalesce(logs, '') = %s" + ).format(sql.Identifier(schema, "_prisma_migrations")), + (progress.id, progress.checksum, progress.logs), + ) + if result.rowcount != 1: + raise RuntimeError("Could not complete the confirmed migration history row; retry startup.") + logger.info("Completed migration %s using its successful SQL step and matching checksum", migration.parent.name) + return True + + +def migration_files(directory: Path) -> tuple[tuple[str, str], ...]: + return tuple( + (path.parent.name, hashlib.sha256(path.read_bytes()).hexdigest()) + for path in sorted((directory / "migrations").glob("*/migration.sql")) + ) + + +def baseline_current_schema( + coordinator: MigrationCoordinator, + schema: str, + migrations_dir: Path, + prisma_command: str, + prisma_env: Mapping[str, str], +) -> None: + from psycopg import sql + + packaged_dir: Final = Path(__file__).parent + migrations: Final = migration_files(migrations_dir) + if ( + not migrations + or migrations != migration_files(packaged_dir) + or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes() + ): + raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.") + + coordinator.acquire_prisma_lock() + existing: Final = coordinator.connection.execute( + "SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(coordinator.connection),) + ).fetchone() + if existing is not None and existing[0] is not None: + return + try: + prisma_toolchain.run_prisma( + ( + prisma_command, + "migrate", + "diff", + "--from-schema-datasource", + str(migrations_dir / "schema.prisma"), + "--to-schema-datamodel", + str(migrations_dir / "schema.prisma"), + "--exit-code", + ), + timeout=prisma_toolchain.prisma_command_timeout(), + env=prisma_env, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + raise RuntimeError( + "Cannot automatically baseline this database: its schema has not been verified to match this build. " + "Establish the existing migration history before retrying. No schema reconciliation was performed. " + "If using a transaction pooler, configure DIRECT_URL to reach the same database without the pooler. " + f"Schema verification detail: {exc.stderr}" + ) from exc + + coordinator.check_connection() + ledger: Final = sql.Identifier(schema, "_prisma_migrations") + coordinator.connection.execute( + sql.SQL( + "CREATE TABLE {} (id varchar(36) PRIMARY KEY NOT NULL, checksum varchar(64) NOT NULL, " + "finished_at timestamptz, migration_name varchar(255) NOT NULL, logs text, rolled_back_at timestamptz, " + "started_at timestamptz NOT NULL DEFAULT now(), applied_steps_count integer NOT NULL DEFAULT 0)" + ).format(ledger) + ) + with coordinator.connection.cursor() as cursor: + cursor.executemany( + sql.SQL( + "INSERT INTO {} (id, checksum, migration_name, logs, started_at, finished_at) " + "VALUES (%s, %s, %s, '', current_timestamp, current_timestamp)" + ).format(ledger), + tuple((str(uuid4()), checksum, name) for name, checksum in migrations), + ) + logger.warning( + "Legacy migration history was missing. The existing Prisma schema matches this build; " + "adopted %s packaged migrations as a baseline. No schema changes were applied, and " + "historical data backfills were not replayed or verified. Continuing startup; " + "review any feature-specific backfill requirements.", + len(migrations), + ) + + +_LINE_COMMENT_RE: Final = re.compile(r"--[^\n]*") +_BLOCK_COMMENT_RE: Final = re.compile(r"/\*.*?\*/", re.DOTALL) +_NO_OP_STATEMENT_RE: Final = re.compile(r"^\s*SELECT\s+1\s*$", re.IGNORECASE) + + +def is_inert_migration(script: str) -> bool: + """Whether a migration file changes nothing: only comments and `SELECT 1`, so + applying it can neither repeat nor skip a database change.""" + stripped: Final = _LINE_COMMENT_RE.sub("", _BLOCK_COMMENT_RE.sub("", script)) + return all(not part.strip() or _NO_OP_STATEMENT_RE.match(part) for part in stripped.split(";")) + + +def roll_back_failed_inert_migration(coordinator: MigrationCoordinator, schema: str, migration: Path) -> bool: + """Roll back the failed ledger row of a migration whose file in this build is inert, + so `migrate deploy` applies the inert file on its next pass. The row records an + earlier build's attempt at SQL this build no longer ships (an index now built by the + migration job), so no database change can be repeated or skipped by replaying + the empty file. The caller commits this checkpoint before the next Prisma command. + """ + from psycopg import sql + + if not is_inert_migration(migration.read_text(encoding="utf-8")): + return False + coordinator.acquire_prisma_lock() + records: Final = _migration_records(coordinator.connection, schema, migration) + unfinished: Final = tuple(record for record in records if not record.finished) + if len(unfinished) != 1: + return False + result: Final = coordinator.connection.execute( + sql.SQL( + "UPDATE {} SET rolled_back_at = current_timestamp " + "WHERE id = %s AND finished_at IS NULL AND rolled_back_at IS NULL" + ).format(sql.Identifier(schema, "_prisma_migrations")), + (unfinished[0].id,), + ) + if result.rowcount != 1: + raise RuntimeError("Could not roll back the failed inert migration history row; rerun the database setup.") + logger.info( + "Rolled back the failed history row of %s: this build ships it as an inert migration, " + "its index is built by the migration job", + migration.parent.name, + ) + return True diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260823000000_add_spend_logs_api_key_starttime_index/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260823000000_add_spend_logs_api_key_starttime_index/migration.sql index e0ac49d1ecf..a2bec81ca00 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260823000000_add_spend_logs_api_key_starttime_index/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260823000000_add_spend_logs_api_key_starttime_index/migration.sql @@ -1 +1,6 @@ +-- The (api_key, startTime) index on LiteLLM_SpendLogs is built after migrate deploy, +-- through litellm_proxy_extras/request_log_indexes.py: concurrently on a plain table and +-- per partition on a partitioned one. The migration job builds it; a serving proxy that +-- ran the migrations itself builds it in the background once it serves. A migration +-- cannot do either without blocking spend-log writes or failing on a partitioned table. SELECT 1; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120001_spend_logs_litellm_call_id_index/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120001_spend_logs_litellm_call_id_index/migration.sql index e0ac49d1ecf..7eba7fc9b97 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120001_spend_logs_litellm_call_id_index/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120001_spend_logs_litellm_call_id_index/migration.sql @@ -1 +1,6 @@ +-- The litellm_call_id index on LiteLLM_SpendLogs is built after migrate deploy, through +-- litellm_proxy_extras/request_log_indexes.py: concurrently on a plain table and per +-- partition on a partitioned one. The migration job builds it; a serving proxy that ran +-- the migrations itself builds it in the background once it serves. Postgres refuses +-- CREATE INDEX CONCURRENTLY on a partitioned parent, so this migration no longer runs it. SELECT 1; diff --git a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py index 9cd48fcf11a..07f83f76d2b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py +++ b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py @@ -59,6 +59,7 @@ except ImportError: PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT" PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT" PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT" +MIGRATION_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_LOCK_TIMEOUT" NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR" DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0 @@ -106,6 +107,10 @@ def prisma_command_timeout() -> float: ) +def migration_lock_timeout() -> float: + return _timeout_from_env(MIGRATION_LOCK_TIMEOUT_ENV_VAR, 600.0) + + def prisma_bootstrap_timeout() -> float: """Seconds the one-time Node toolchain install may run for.""" return _timeout_from_env( diff --git a/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py b/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py new file mode 100644 index 00000000000..ec0e5368ab8 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py @@ -0,0 +1,480 @@ +"""The request-log indexes built after `prisma migrate deploy` instead of by a migration: +by the migration job, or by a serving proxy that ran the migrations itself (in the +background, once it serves). + +A migration cannot build them: a plain `CREATE INDEX` blocks spend-log inserts for the +whole build, and `CREATE INDEX CONCURRENTLY` is refused on a partitioned parent +(db_scripts/partition_spend_logs.sql). `REQUEST_LOG_INDEXES` is the one list to extend; +names match what Prisma derives from the `@@index` declarations in schema.prisma, so an +index a database already has is recognized and never rebuilt. +""" + +import hashlib +import random +import re +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final + +from litellm_proxy_extras._logging import logger +from litellm_proxy_extras.migration_lock import held_migration_lock + +if TYPE_CHECKING: + import psycopg + from psycopg import sql + + +@dataclass(frozen=True, slots=True) +class RequestLogIndex: + """One index the migration job owns: the table, the exact Prisma index name and the + column list as it would be written after `ON `.""" + + table: str + name: str + definition: str + + @property + def columns(self) -> tuple[str, ...]: + return tuple(re.findall(r'"([^"]+)"', self.definition)) + + def partition_index_name(self, partition: str) -> str: + """The child index name for one partition, built the way Postgres names the + children of a partitioned index, and kept within the 63 byte identifier limit.""" + name: Final = f"{partition}_{self.name.removeprefix(f'{self.table}_')}" + if len(name.encode()) <= _IDENTIFIER_MAX_BYTES: + return name + digest: Final = hashlib.sha256(name.encode()).hexdigest()[:_DIGEST_LENGTH] + budget: Final = _IDENTIFIER_MAX_BYTES - _DIGEST_LENGTH - 1 + kept: Final = next(name[:length] for length in range(len(name), 0, -1) if len(name[:length].encode()) <= budget) + return f"{kept}_{digest}" + + +REQUEST_LOG_INDEXES: Final = ( + RequestLogIndex("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", '("api_key", "startTime")'), + RequestLogIndex("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_litellm_call_id_idx", '("litellm_call_id")'), +) + +_IDENTIFIER_MAX_BYTES: Final = 63 +_DDL_LOCK_TIMEOUT: Final = "200ms" +_DDL_LOCK_ATTEMPTS: Final = 10 +_DDL_RETRY_BASE_SECONDS: Final = 0.25 +_DDL_RETRY_MAX_SECONDS: Final = 8.0 +_LOCK_HANDOVER_SECONDS: Final = 2.0 +_DIGEST_LENGTH: Final = 8 +_CREATE_INDEX_STATEMENT: Final = re.compile( + r'^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?"(?P[^"]+)"\s+ON\b', + re.IGNORECASE, +) +_TABLE_KIND_SQL: Final = "SELECT c.relkind = 'p' AS partitioned FROM pg_class c WHERE c.oid = to_regclass(%s)" +_REQUEST_LOG_TABLES_SQL: Final = ( + "WITH RECURSIVE tables AS (" + "SELECT c.oid, c.relname, c.relnamespace FROM pg_class c " + "JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = %s AND c.relname = ANY(%s) " + "UNION SELECT c.oid, c.relname, c.relnamespace FROM tables p " + "JOIN pg_inherits h ON h.inhparent = p.oid JOIN pg_class c ON c.oid = h.inhrelid" + ") SELECT t.relname FROM tables t JOIN pg_namespace n ON n.oid = t.relnamespace WHERE n.nspname = %s" +) +_CHILDREN_WITHOUT_THE_INDEX_SQL: Final = ( + "SELECT child.relname AS name, n.nspname AS schema, child.relkind = 'p' AS partitioned " + "FROM pg_inherits i JOIN pg_class child ON child.oid = i.inhrelid " + "JOIN pg_namespace n ON n.oid = child.relnamespace " + "WHERE i.inhparent = to_regclass(%s) AND NOT EXISTS (" + "SELECT 1 FROM pg_inherits attached JOIN pg_index x ON x.indexrelid = attached.inhrelid " + "WHERE attached.inhparent = to_regclass(%s) AND x.indrelid = child.oid) " + "ORDER BY child.relname" +) +_EQUIVALENT_INDEXES_SQL: Final = ( + "SELECT i.relname AS name, x.indisvalid AS valid " + "FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid JOIN pg_am am ON am.oid = i.relam " + "WHERE x.indrelid = to_regclass(%s) AND i.relname <> %s AND am.amname = 'btree' AND NOT x.indisunique " + "AND x.indexprs IS NULL AND x.indpred IS NULL AND cardinality(x.indclass::oid[]) = x.indnatts " + "AND NOT EXISTS (SELECT 1 FROM unnest(x.indoption::int2[]) o WHERE o <> 0) " + "AND NOT EXISTS (SELECT 1 FROM unnest(x.indclass::oid[]) c JOIN pg_opclass oc ON oc.oid = c WHERE NOT oc.opcdefault) " + "AND NOT EXISTS (SELECT 1 FROM unnest(x.indcollation::oid[]) WITH ORDINALITY c(coll, ord) " + "JOIN unnest(x.indkey::int2[]) WITH ORDINALITY k(attnum, ord) ON k.ord = c.ord " + "JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + "WHERE c.coll <> 0 AND c.coll <> a.attcollation) " + "AND (SELECT array_agg(a.attname::text ORDER BY k.ord) FROM unnest(x.indkey::int2[]) WITH ORDINALITY k(attnum, ord) " + "JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum) = %s::text[] " + "AND NOT EXISTS (SELECT 1 FROM pg_inherits WHERE inhrelid = x.indexrelid) " + "ORDER BY x.indisvalid DESC, i.relname" +) +_INDEX_STATE_SQL: Final = ( + 'SELECT x.indisvalid AS valid, t.relname AS "table" ' + "FROM pg_index x JOIN pg_class t ON t.oid = x.indrelid WHERE x.indexrelid = to_regclass(%s)" +) + + +@dataclass(frozen=True, slots=True) +class _Relation: + name: str + schema: str + partitioned: bool + + +@dataclass(frozen=True, slots=True) +class _IndexState: + valid: bool + table: str + + +@dataclass(frozen=True, slots=True) +class _EquivalentIndex: + name: str + valid: bool + + +@dataclass(frozen=True, slots=True) +class _TableKind: + partitioned: bool + + +def filter_request_log_index_diff(diff_sql: str, indexes: tuple[RequestLogIndex, ...] = REQUEST_LOG_INDEXES) -> str: + """The `prisma migrate diff` script without the statements that create a migration-job-owned + index, which the schema declares and the migrations deliberately do not build.""" + names: Final = frozenset(index.name for index in indexes) + statements: Final = diff_sql.split(";") + kept: Final = tuple(statement for statement in statements if not _creates_one_of(statement, names)) + return ";".join(kept) if any(part.strip() for part in kept) else "" + + +def _creates_one_of(statement: str, names: frozenset[str]) -> bool: + match: Final = _CREATE_INDEX_STATEMENT.match(_without_comments(statement)) + return match is not None and match["index"] in names + + +def _without_comments(statement: str) -> str: + return "\n".join(line for line in statement.splitlines() if not line.lstrip().startswith("--")) + + +def _connect(database_url: str) -> "psycopg.Connection[tuple[object, ...]]": + import psycopg + + return psycopg.connect(database_url, connect_timeout=10, autocommit=True) + + +def ensure_request_log_indexes( + database_url: str, + schema: str, + indexes: tuple[RequestLogIndex, ...] = REQUEST_LOG_INDEXES, + connect: "Callable[[str], psycopg.Connection[tuple[object, ...]]]" = _connect, +) -> bool: + """Build every listed index that is missing or invalid. Each build step runs under + the migration coordinator lock, held per statement so a resolver booting on another + replica gets in between partitions rather than waiting for the whole table. Any + failure is logged and left for the next index build; the result says whether + every index ended up valid. Never raises.""" + import psycopg + + try: + with connect(database_url) as connection: + connection.execute("SET statement_timeout = 0") + results: Final = tuple(_ensure_index(connection, schema, index) for index in indexes) + except psycopg.Error as exc: + logger.warning("Could not build the request-log indexes, leaving them for the next index build: %s", exc) + return False + if not all(results): + logger.warning("Some request-log indexes are not in place yet, leaving them for the next index build") + return False + logger.info("Request-log indexes are all in place") + return True + + +def request_log_index_names(connection: "psycopg.Connection[tuple[object, ...]]", schema: str) -> frozenset[str]: + tables: Final = connection.execute( + _REQUEST_LOG_TABLES_SQL, (schema, sorted(frozenset(index.table for index in REQUEST_LOG_INDEXES)), schema) + ).fetchall() + return frozenset(index.name for index in REQUEST_LOG_INDEXES) | frozenset( + index.partition_index_name(str(table[0])) for table in tables for index in REQUEST_LOG_INDEXES + ) + + +def _under_migration_lock(connection: "psycopg.Connection[tuple[object, ...]]", step: Callable[[], bool]) -> bool: + with held_migration_lock(connection) as held: + if not held: + logger.info( + "Another process holds the migration lock, leaving the request-log indexes to the next index build" + ) + return False + return step() + + +def _with_bounded_lock( + connection: "psycopg.Connection[tuple[object, ...]]", step: Callable[[], bool], what: str +) -> bool: + """Run `step` under the migration lock with a short lock_timeout, so a DDL statement that has to wait for open + transactions holds new writes back for at most that long; retry with capped exponential backoff, holding the + migration lock per attempt only and releasing it while sleeping. False when another process holds the migration + lock or every attempt timed out.""" + import psycopg + from psycopg import sql + + for attempt in range(_DDL_LOCK_ATTEMPTS): + if attempt: + time.sleep(min(_DDL_RETRY_MAX_SECONDS, _DDL_RETRY_BASE_SECONDS * 2.0**attempt) * random.uniform(0.5, 1.0)) + connection.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(_DDL_LOCK_TIMEOUT))) + try: + return _under_migration_lock(connection, step) + except psycopg.errors.LockNotAvailable: + logger.info("Waiting for open transactions before %s", what) + finally: + connection.execute("SET lock_timeout = 0") + logger.warning( + "Could not get the lock for %s without holding writes back, leaving it for the next index build", what + ) + return False + + +def _ensure_index(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: RequestLogIndex) -> bool: + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(_TableKind)) as cursor: + table: Final = cursor.execute(_TABLE_KIND_SQL, (_regclass_name(connection, schema, index.table),)).fetchone() + if table is None: + logger.info("Table %s does not exist yet, skipping index %s", index.table, index.name) + return True + if table.partitioned: + return build_index_on_partitioned_table(connection, schema, index) + return _build_leaf_index(connection, schema, index.table, index.name, index) + + +def _regclass_name(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, name: str) -> str: + from psycopg import sql + + return sql.Identifier(schema, name).as_string(connection) + + +def _create_index_statement( + connection: "psycopg.Connection[tuple[object, ...]]", prefix: "sql.Composed", definition: str +) -> bytes: + return (prefix.as_string(connection) + definition).encode() + + +def _index_state(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: str) -> "_IndexState | None": + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(_IndexState)) as cursor: + return cursor.execute(_INDEX_STATE_SQL, (_regclass_name(connection, schema, index),)).fetchone() + + +def _equivalent_indexes( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + table: str, + name: str, + index: RequestLogIndex, +) -> tuple[_EquivalentIndex, ...]: + """The indexes on `table` other than `name` with the same definition: default btree + over the same columns in the same order, no expression, predicate, DESC or custom + opclass or collation, and not attached under a partitioned index. Valid ones first.""" + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(_EquivalentIndex)) as cursor: + return tuple( + cursor.execute( + _EQUIVALENT_INDEXES_SQL, (_regclass_name(connection, schema, table), name, list(index.columns)) + ).fetchall() + ) + + +def _adopt_equivalent_index( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + table: str, + name: str, + index: RequestLogIndex, +) -> bool: + """Rename a valid index of the same definition under another name (an operator's + hand-built copy, say) to the name this code expects, instead of building a second + one. RENAME on an index is a catalog change that lets writes through.""" + from psycopg import sql + + equivalent: Final = next( + (found for found in _equivalent_indexes(connection, schema, table, name, index) if found.valid), None + ) + if equivalent is None: + return False + logger.info( + "Renaming the equivalent index %s on %s to %s instead of building a second one", equivalent.name, table, name + ) + connection.execute( + sql.SQL("ALTER INDEX {} RENAME TO {}").format(sql.Identifier(schema, equivalent.name), sql.Identifier(name)) + ) + return True + + +def _report_second_copies( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + table: str, + name: str, + index: RequestLogIndex, + concurrently: bool, +) -> None: + """Log every other index of the same definition with the statement that removes it. + Dropping is the operator's call: a second copy costs writes and disk, never results.""" + from psycopg import sql + + drop: Final = "DROP INDEX CONCURRENTLY" if concurrently else "DROP INDEX" + for copy in _equivalent_indexes(connection, schema, table, name, index): + logger.warning( + "Index %s on %s is a second copy of %s and only costs writes and disk; remove it with: %s %s", + copy.name, + table, + name, + drop, + sql.Identifier(schema, copy.name).as_string(connection), + ) + + +def _children_without_the_index( + connection: "psycopg.Connection[tuple[object, ...]]", schema: str, table: str, index: str +) -> tuple[_Relation, ...]: + from psycopg.rows import class_row + + with connection.cursor(row_factory=class_row(_Relation)) as cursor: + return tuple( + cursor.execute( + _CHILDREN_WITHOUT_THE_INDEX_SQL, + (_regclass_name(connection, schema, table), _regclass_name(connection, schema, index)), + ).fetchall() + ) + + +def _build_leaf_index( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + table: str, + name: str, + index: RequestLogIndex, +) -> bool: + """Build one plain table's or partition's index with CONCURRENTLY so writes keep + flowing. The catalog is read under the migration lock, so a replica that saw an + invalid index before the lock finds the valid one another replica just built and + leaves it. An invalid index left by an interrupted build is dropped and rebuilt; a + valid index of the same definition under another name is renamed rather than + duplicated; an index of that name on another table is a collision this code will + not touch.""" + from psycopg import sql + + def build() -> bool: + existing: Final = _index_state(connection, schema, name) + if existing is not None and existing.table != table: + logger.warning( + "Index %s already exists on %s rather than %s, leaving it alone", name, existing.table, table + ) + return False + if existing is not None and existing.valid: + return True + if existing is not None: + logger.info("Dropping the invalid index %s left by an interrupted build on %s", name, table) + connection.execute(sql.SQL("DROP INDEX CONCURRENTLY {}").format(sql.Identifier(schema, name))) + elif _adopt_equivalent_index(connection, schema, table, name, index): + return True + logger.info("Building index %s on %s concurrently", name, table) + prefix: Final = sql.SQL("CREATE INDEX CONCURRENTLY IF NOT EXISTS {} ON {} ").format( + sql.Identifier(name), sql.Identifier(schema, table) + ) + connection.execute(_create_index_statement(connection, prefix, index.definition)) + built: Final = _index_state(connection, schema, name) + return built is not None and built.valid + + current: Final = _index_state(connection, schema, name) + if current is None or not current.valid or current.table != table: + if not _under_migration_lock(connection, build): + return False + time.sleep(_LOCK_HANDOVER_SECONDS) + _report_second_copies(connection, schema, table, name, index, concurrently=True) + return True + + +def build_index_on_partitioned_table( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + index: RequestLogIndex, + table: "str | None" = None, + name: "str | None" = None, +) -> bool: + """Build the index the way Postgres allows on a partitioned parent: a metadata-only + parent index ON ONLY the parent, one CONCURRENTLY build per partition, and ATTACH + PARTITION for each child. Partitions that are themselves partitioned get the same + treatment one level down. Every step checks the catalog before acting, so an + interrupted run resumes where it stopped and a second run finds nothing to do; a + parent or child index of the same definition under another name is renamed and + used rather than duplicated. The connection must be in autocommit mode. True when + the parent index ends up valid.""" + + parent_table: Final = index.table if table is None else table + parent_index: Final = index.name if name is None else name + existing: Final = _index_state(connection, schema, parent_index) + if existing is not None and existing.table != parent_table: + logger.warning( + "Index %s already exists on %s rather than %s, leaving it alone", parent_index, existing.table, parent_table + ) + return False + if existing is None and not _with_bounded_lock( + connection, + lambda: ( + _adopt_equivalent_index(connection, schema, parent_table, parent_index, index) + or _create_parent_index(connection, schema, parent_index, parent_table, index) + ), + f"creating the parent index {parent_index}", + ): + return False + children: Final = _children_without_the_index(connection, schema, parent_table, parent_index) + if not all(_attach_child_index(connection, schema, parent_index, child, index) for child in children): + return False + final: Final = _index_state(connection, schema, parent_index) + if final is None or not final.valid: + return False + _report_second_copies(connection, schema, parent_table, parent_index, index, concurrently=False) + return True + + +def _create_parent_index( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + name: str, + table: str, + index: RequestLogIndex, +) -> bool: + """Create the metadata-only parent index. The caller bounds Postgres's SHARE lock wait on the parent.""" + from psycopg import sql + + prefix: Final = sql.SQL("CREATE INDEX IF NOT EXISTS {} ON ONLY {} ").format( + sql.Identifier(name), sql.Identifier(schema, table) + ) + statement: Final = _create_index_statement(connection, prefix, index.definition) + connection.execute(statement) + return True + + +def _attach_child_index( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + parent_index: str, + child: _Relation, + index: RequestLogIndex, +) -> bool: + from psycopg import sql + + child_index: Final = index.partition_index_name(child.name) + built: Final = ( + build_index_on_partitioned_table(connection, child.schema, index, child.name, child_index) + if child.partitioned + else _build_leaf_index(connection, child.schema, child.name, child_index, index) + ) + if not built: + return False + + def attach() -> bool: + connection.execute( + sql.SQL("ALTER INDEX {} ATTACH PARTITION {}").format( + sql.Identifier(schema, parent_index), sql.Identifier(child.schema, child_index) + ) + ) + logger.info("Attached index %s on partition %s to %s", child_index, child.name, parent_index) + return True + + return _with_bounded_lock(connection, attach, f"attaching {child_index}") diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index b44cc1d98f9..fbc46ac0ee3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -5,13 +5,16 @@ import re import shutil import subprocess import tempfile +import threading import time +from collections.abc import Callable from dataclasses import dataclass, replace from pathlib import Path from typing import TYPE_CHECKING, Final, Optional from litellm_proxy_extras import prisma_toolchain from litellm_proxy_extras._logging import logger +from litellm_proxy_extras.migration_lock import held_migration_lock from litellm_proxy_extras.prisma_toolchain import ( PRISMA_COMMAND_TIMEOUT_ENV_VAR, PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, @@ -23,6 +26,11 @@ from litellm_proxy_extras.replica_identity import ( REPLICA_IDENTITY_FULL_ENV_VAR, apply_replica_identity_full, ) +from litellm_proxy_extras.request_log_indexes import ( + ensure_request_log_indexes, + filter_request_log_index_diff, + request_log_index_names, +) if TYPE_CHECKING: import psycopg @@ -74,19 +82,31 @@ class _InvalidIndex: table_size: str MAX_MIGRATE_DEPLOY_ATTEMPTS = 4 +LIBPQ_URL_PARAMS: Final = frozenset( + { + "sslmode", + "sslcert", + "sslkey", + "sslrootcert", + "sslpassword", + "application_name", + "connect_timeout", + "client_encoding", + "options", + "service", + "gssencmode", + "krbsrvname", + "target_session_attrs", + } +) @dataclass(frozen=True) class _MigrateAttemptBudget: - """Retries left, and the recoveries already run. - - A recovery that lands something new costs nothing, so a database full of - objects `prisma db push` created works through them one per pass. Anything - that made no progress spends an attempt, so a stuck run still gives up. - """ + """Independent bounds for failed attempts and Prisma lock contention.""" attempts_left: int - recoveries: frozenset[str] = frozenset() + contention_seconds_left: float = 600.0 @property def exhausted(self) -> bool: @@ -99,10 +119,14 @@ class _MigrateAttemptBudget: def spend(self) -> "_MigrateAttemptBudget": return replace(self, attempts_left=self.attempts_left - 1) - def after_recovery(self, recovery: str) -> "_MigrateAttemptBudget": - if recovery in self.recoveries: - return self.spend() - return replace(self, recoveries=self.recoveries | {recovery}) + def after_contention(self, elapsed: float) -> "_MigrateAttemptBudget": + remaining: Final = self.contention_seconds_left - elapsed + if remaining <= 0: + raise RuntimeError( + "Timed out waiting for Prisma's migration advisory lock. Check the running migration " + "or increase LITELLM_MIGRATION_LOCK_TIMEOUT." + ) + return replace(self, contention_seconds_left=remaining) _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) @@ -115,17 +139,6 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile( re.IGNORECASE, ) -HAND_BUILT_SPEND_LOGS_INDEXES: Final = frozenset( - { - "LiteLLM_SpendLogs_api_key_startTime_idx", - "LiteLLM_SpendLogs_litellm_call_id_idx", - } -) -_CREATE_INDEX_RE: Final = re.compile( - r'^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?"(?P[^"]+)"\s+ON\b', - re.IGNORECASE, -) - PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), " "so its primary key must include the partition key (\"startTime\"). `prisma db push` " @@ -176,21 +189,6 @@ def filter_partitioned_spend_logs_diff(diff_sql: str) -> str: return "".join(f"{statement};\n\n" for statement in kept) -def filter_hand_built_spend_logs_index_diff(diff_sql: str) -> str: - """The `prisma migrate diff` script without the statements that create the - LiteLLM_SpendLogs indexes schema.prisma declares but the migrations deliberately - leave for operators to build online.""" - kept: Final = tuple( - statement for statement in diff_sql.split(";") if not _creates_hand_built_spend_logs_index(statement) - ) - return ";".join(kept) if any(part.strip() for part in kept) else "" - - -def _creates_hand_built_spend_logs_index(statement: str) -> bool: - match: Final = _CREATE_INDEX_RE.match(_without_sql_comments(statement)) - return match is not None and match["index"] in HAND_BUILT_SPEND_LOGS_INDEXES - - def _migration_timestamp(name: str) -> int: """Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name. @@ -359,9 +357,8 @@ class ProxyExtrasDBManager: pass @staticmethod - def _failed_migration_logs(migration_name: str) -> Optional[str]: - """Return failed migration logs, or None if the ledger is unavailable.""" - database_url = os.getenv("DATABASE_URL") + def _read_migration_ledger(query: str, params: tuple[str, ...]) -> "tuple[object, ...] | None": + database_url: Final = os.getenv("DATABASE_URL") if not database_url: return None @@ -370,28 +367,37 @@ class ProxyExtrasDBManager: except ImportError: return None - cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) - ledger_table = psycopg.sql.SQL("{}.{}").format( - psycopg.sql.Identifier( - ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" - ), + cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url) + ledger_table: Final = psycopg.sql.SQL("{}.{}").format( + psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"), psycopg.sql.Identifier("_prisma_migrations"), ) try: - with psycopg.connect( - cleaned_url, connect_timeout=10, autocommit=True - ) as conn: - row = conn.execute( - psycopg.sql.SQL( - "SELECT logs FROM {} " - "WHERE migration_name = %s AND finished_at IS NULL " - "AND rolled_back_at IS NULL" - ).format(ledger_table), - (migration_name,), - ).fetchone() + with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn: + row: Final = conn.execute(psycopg.sql.SQL(query).format(ledger_table), params).fetchone() except (psycopg.OperationalError, psycopg.DatabaseError): return None - return (row[0] or "") if row else "" + return tuple(row) if row is not None else () + + @staticmethod + def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]: + row: Final = ProxyExtrasDBManager._read_migration_ledger( + "SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz " + "AND finished_at IS NULL AND rolled_back_at IS NULL", + (migration_name, started_at), + ) + if row is None: + return None + return row[0] if row and isinstance(row[0], str) else "" + + @staticmethod + def _failed_migration_recovered(migration_name: str, started_at: str) -> bool: + row: Final = ProxyExtrasDBManager._read_migration_ledger( + "SELECT 1 FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz " + "AND (finished_at IS NOT NULL OR rolled_back_at IS NOT NULL)", + (migration_name, started_at), + ) + return bool(row) @staticmethod def _resolve_specific_migration(migration_name: str): @@ -460,12 +466,13 @@ class ProxyExtrasDBManager: return False @staticmethod - def _filter_drift_script(diff_sql: str) -> str: - """The drift script without the hand-built SpendLogs indexes and, when - LiteLLM_SpendLogs is partitioned, without its primary-key rewrite and - partitioning artifacts.""" - without_indexes: Final = filter_hand_built_spend_logs_index_diff(diff_sql) - if not ProxyExtrasDBManager.spend_logs_is_partitioned(): + def _filter_migration_job_owned_drift(diff_sql: str, partitioned: bool | None = None) -> str: + """The drift script without the indexes the migration job builds (the schema + declares them, the migrations deliberately do not) and, when LiteLLM_SpendLogs + is partitioned, without its primary-key rewrite and partitioning artifacts.""" + without_indexes: Final = filter_request_log_index_diff(diff_sql) + is_partitioned: Final = ProxyExtrasDBManager.spend_logs_is_partitioned() if partitioned is None else partitioned + if not is_partitioned: return without_indexes logger.info( "LiteLLM_SpendLogs is partitioned; removed its primary-key " @@ -553,7 +560,7 @@ class ProxyExtrasDBManager: return logger.info(f"Migration diff created at {diff_sql_path}") - filtered_sql: Final = ProxyExtrasDBManager._filter_drift_script(diff_sql_path.read_text()) + filtered_sql: Final = ProxyExtrasDBManager._filter_migration_job_owned_drift(diff_sql_path.read_text()) diff_sql_path.write_text(filtered_sql) if not filtered_sql.strip(): logger.info("Drift script is empty after filtering; nothing to apply") @@ -681,30 +688,50 @@ class ProxyExtrasDBManager: @staticmethod def _strip_prisma_query_params(url: str) -> str: - """Remove Prisma-specific query params (connection_limit, pool_timeout, - schema, etc.) from DATABASE_URL so psycopg can parse it.""" - from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse + """Rewrite a Prisma-dialect URL for libpq: drop the Prisma-only params + (connection_limit, pool_timeout, schema, pgbouncer, sslaccept, ...) and + translate Prisma's TLS params back, since libpq reads ``sslcert`` as a + client certificate where Prisma reads it as the CA.""" + from urllib.parse import parse_qsl, quote, unquote, urlencode, urlparse, urlunparse - parsed = urlparse(url) + parsed: Final = urlparse(url) + netloc: Final = parsed.netloc + userinfo, separator, hostport = netloc.rpartition("@") + user, password_separator, password = userinfo.partition(":") + canonical_userinfo: Final = quote(unquote(user), safe="") + ( + password_separator + quote(unquote(password), safe="") if password_separator else "" + ) + canonical_netloc: Final = f"{canonical_userinfo}{separator}{hostport}" if separator else netloc if not parsed.query: - return url - libpq_params = { - "sslmode", - "sslcert", - "sslkey", - "sslrootcert", - "sslpassword", - "application_name", - "connect_timeout", - "client_encoding", - "options", - "service", - "gssencmode", - "krbsrvname", - "target_session_attrs", - } - kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params] - return urlunparse(parsed._replace(query=urlencode(kept, quote_via=quote))) + return url if canonical_netloc == netloc else urlunparse(parsed._replace(netloc=canonical_netloc)) + pairs: Final = tuple(parse_qsl(parsed.query)) + kept: Final = tuple((k, v) for k, v in pairs if k in LIBPQ_URL_PARAMS) + sslaccept: Final = next((v for k, v in pairs if k == "sslaccept"), None) + libpq_pairs: Final = ProxyExtrasDBManager._libpq_tls_params(kept, sslaccept) + return urlunparse(parsed._replace(netloc=canonical_netloc, query=urlencode(libpq_pairs, quote_via=quote))) + + @staticmethod + def _libpq_tls_params( + pairs: "tuple[tuple[str, str], ...]", sslaccept: "str | None" + ) -> "tuple[tuple[str, str], ...]": + """Undo ``translate_libpq_ssl_params``. Prisma's ``sslcert`` is the CA and + ``sslaccept=strict`` checks chain and hostname, which libpq only does in + ``sslmode=verify-full``, so strict becomes ``sslrootcert`` plus + ``verify-full`` whatever ``sslmode`` said (``disable`` stays off). Prisma + defaults an absent ``sslaccept`` to ``accept_invalid_certs`` and anything + else to strict. Without strict it checks nothing, so the CA is dropped and + ``sslmode`` is kept as is: libpq only verifies when a root cert is present. + A URL that also carries ``sslkey`` is libpq's own client-certificate form + and is kept.""" + keys: Final = frozenset(k for k, _ in pairs) + if "sslcert" not in keys or "sslkey" in keys: + return pairs + sslmode: Final = next((v for k, v in pairs if k == "sslmode"), None) + rest: Final = tuple((k, v) for k, v in pairs if k not in ("sslcert", "sslmode")) + if sslaccept in (None, "accept_invalid_certs") or sslmode == "disable": + return rest if sslmode is None else rest + (("sslmode", sslmode),) + root_cert: Final = tuple(("sslrootcert", v) for k, v in pairs if k == "sslcert" and "sslrootcert" not in keys) + return rest + root_cert + (("sslmode", "verify-full"),) @staticmethod def _warn_if_db_ahead_of_head(migrations_dir: str) -> None: @@ -803,7 +830,7 @@ class ProxyExtrasDBManager: conn.execute(statement) except psycopg.Error as e: logger.warning( - "Could not repair invalid index %s.%s, will retry on the next startup. " + "Could not repair invalid index %s.%s, will retry on the next database setup run. " "If this keeps happening, run `%s` by hand as the index owner. Error: %s", index.schema, index.name, @@ -814,16 +841,24 @@ class ProxyExtrasDBManager: logger.info("%s invalid index %s.%s", action, index.schema, index.name) @staticmethod - def repair_invalid_indexes(lock_timeout: str = "30s") -> bool: + def repair_invalid_indexes( + lock_timeout: str = "30s", + repair: "Callable[[psycopg.Connection[tuple[str, str, str]], _InvalidIndex], None] | None" = None, + *, + skip_request_log_indexes: bool = False, + ) -> bool: """Rebuild LiteLLM indexes an interrupted CREATE INDEX CONCURRENTLY left INVALID (a migration deadlock between replicas is the usual cause; the retried migration skips them because of IF NOT EXISTS). Never raises: returns True when no invalid index remains, False when the repair was - skipped or failed and will be retried on the next startup. Looks in the + skipped or failed and will be retried on the next database setup run. Looks in the schema DATABASE_URL names, the only URL Prisma migrates through, but connects over DIRECT_URL when set: the session settings, the advisory lock and REINDEX CONCURRENTLY all need one server session, which a - transaction pooler does not give.""" + transaction pooler does not give. Each rebuild holds the migration + coordinator lock on its own, like the migration job's index build, so a resolver + booting on another replica waits for one index at most. Database setup skips + managed request-log indexes so their builder owns repair without delaying serving startup.""" prisma_url: Final = os.getenv("DATABASE_URL") if not prisma_url: return False @@ -844,7 +879,11 @@ class ProxyExtrasDBManager: with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn: conn.execute("SET statement_timeout = 0") conn.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(lock_timeout))) - found: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema) + deferred: Final = request_log_index_names(conn, schema) if skip_request_log_indexes else frozenset() + found: Final = tuple( + index for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema) + if index.name not in deferred + ) if not found: return True logger.warning( @@ -859,22 +898,95 @@ class ProxyExtrasDBManager: if lock_row is None or not lock_row[0]: logger.info("Another replica is already rebuilding the invalid indexes, skipping") return False - for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema): - ProxyExtrasDBManager._repair_index(conn, index) - remaining: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema) + repair_one: Final = repair or ProxyExtrasDBManager._repair_index + repaired: Final = all( + ProxyExtrasDBManager._repair_under_migration_lock(conn, schema, index, repair_one) + for index in found + ) + if not repaired: + return False + remaining: Final = tuple( + index for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema) + if index.name not in deferred + ) except psycopg.Error as e: - logger.warning("Could not check for invalid indexes, will retry on the next startup. Error: %s", e) + logger.warning( + "Could not check for invalid indexes, will retry on the next database setup run. Error: %s", e + ) return False return not remaining + @staticmethod + def _repair_under_migration_lock( + conn: "psycopg.Connection[tuple[str, str, str]]", + schema: str, + index: _InvalidIndex, + repair: "Callable[[psycopg.Connection[tuple[str, str, str]], _InvalidIndex], None]", + ) -> bool: + """Rebuild one index under the migration coordinator lock, skipping it when a + migration job finished or dropped it in the meantime. False when another process + holds the lock, so the check waits for the next database setup run.""" + with held_migration_lock(conn) as held: + if not held: + logger.info( + "Another process is building indexes under the migration lock, leaving the " + "invalid index check to the next database setup run" + ) + return False + still_invalid: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema) + if any(found.schema == index.schema and found.name == index.name for found in still_invalid): + repair(conn, index) + return True + @staticmethod def _setup_database_v2(use_migrate: bool) -> bool: + if not use_migrate: + return ProxyExtrasDBManager._run_database_v2(False) + from litellm_proxy_extras.migration_lock import migration_environment, migration_lock + from litellm_proxy_extras.migration_recovery import ( + baseline_current_schema, + recover_completed_migration, + roll_back_failed_inert_migration, + ) + + database_url: Final = os.environ.get("DATABASE_URL") + if not database_url: + raise RuntimeError("DATABASE_URL is required for v2 migrations") + lock_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(os.environ.get("DIRECT_URL") or database_url) + schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" + + def recover_completed(name: str) -> bool: + if Path(name).name != name or "\\" in name: + return False + migration: Final = Path(os.getcwd()) / "migrations" / name / "migration.sql" + if not migration.is_file(): + return False + with migration_lock(lock_url) as coordinator: + return recover_completed_migration(coordinator, schema, migration) or roll_back_failed_inert_migration( + coordinator, schema, migration + ) + + def baseline_existing(migrations_dir: str) -> None: + with migration_lock(lock_url) as coordinator: + baseline_current_schema( + coordinator, schema, Path(migrations_dir), _get_prisma_command(), migration_environment(_get_prisma_env()) + ) + + while not ProxyExtrasDBManager._run_database_v2(True, recover_completed, baseline_existing): + continue + return True + + @staticmethod + def _run_database_v2( + use_migrate: bool, + recover_completed: Callable[[str], bool] = lambda name: False, + baseline_existing: "Callable[[str], None] | None" = None, + ) -> bool: """ v2 migration resolver (opt-in via --use_v2_migration_resolver). - Runs `prisma migrate deploy` and handles standard recovery paths - (P3005 baseline, P3009/P3018 idempotent errors, deadlocks against a - concurrent migrate deploy). Critically, it does + Runs `prisma migrate deploy`, baselines verified existing schemas, + and recovers confirmed SQL completion or reported deadlocks. It does NOT call `_resolve_all_migrations` — the diff-and-force recovery that caused schema thrashing when two LiteLLM versions contended for the same DB during rolling deploys. @@ -883,10 +995,9 @@ class ProxyExtrasDBManager: is logged as a warning, not a fatal error — users whose DBs got into weird shapes from the old thrashing should still be able to start. - The retry budget only counts attempts that made no progress: see - _MigrateAttemptBudget. + False requests a committed recovery checkpoint and another deploy + pass. True means every pending migration is complete. """ - schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma" migrations_dir = ProxyExtrasDBManager._get_prisma_dir() if not use_migrate: @@ -919,14 +1030,22 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) deploy_timeout = prisma_migrate_deploy_timeout() - budget = _MigrateAttemptBudget(attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS) + from litellm_proxy_extras.migration_lock import migration_environment, migration_lock_timeout + + migration_env: Final = migration_environment(_get_prisma_env()) + + budget = _MigrateAttemptBudget( + attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS, + contention_seconds_left=migration_lock_timeout(), + ) try: while not budget.exhausted: + attempt_started = time.monotonic() try: result = prisma_toolchain.run_prisma( [_get_prisma_command(), "migrate", "deploy"], timeout=deploy_timeout, - env=_get_prisma_env(), + env=migration_env, ) logger.info(f"prisma migrate deploy stdout: {result.stdout}") return True @@ -942,8 +1061,16 @@ class ProxyExtrasDBManager: next_budget = budget.spend() except subprocess.CalledProcessError as e: + if "P3005" in (e.stderr or "") and baseline_existing is not None: + baseline_existing(migrations_dir) + return False + failed_migration = ProxyExtrasDBManager._v2_failed_migration_name(e.stderr or "") + if failed_migration and recover_completed(failed_migration): + return False next_budget = ProxyExtrasDBManager._budget_after_deploy_failure( - e, budget, schema_path + e, + budget, + time.monotonic() - attempt_started, ) if next_budget.attempts_left < budget.attempts_left: @@ -952,19 +1079,46 @@ class ProxyExtrasDBManager: raise RuntimeError( f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} " - "attempts that made no progress (timeouts, deadlock retries, or a " - "recovery that had already run once). Check database connectivity, " + "attempts that made no progress (timeouts or deadlock retries). Check database connectivity, " "load, and _prisma_migrations ledger state, and raise " f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out." ) finally: os.chdir(original_dir) + @staticmethod + def _v2_failed_migration_name(stderr: str) -> "str | None": + if "P3009" in stderr: + match = re.search(r"`(\d+_[^`\r\n]+)`", stderr) + return match.group(1) if match else None + if "P3018" in stderr: + match = re.search(r"Migration name: (\d+_[^\r\n]+)", stderr) + return match.group(1) if match else None + return None + + @staticmethod + def _v2_failed_migration_started_at(stderr: str, migration_name: str) -> "str | None": + match: Final = re.search(rf"`{re.escape(migration_name)}` migration started at ([^\r\n]+?) failed", stderr) + return match.group(1) if match else None + + @staticmethod + def _v2_roll_back_migration_best_effort(migration_name: str) -> None: + from litellm_proxy_extras.migration_lock import migration_environment + + try: + prisma_toolchain.run_prisma( + [_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name], + timeout=prisma_command_timeout(), + env=migration_environment(_get_prisma_env()), + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pass + @staticmethod def _budget_after_deploy_failure( error: subprocess.CalledProcessError, budget: "_MigrateAttemptBudget", - schema_path: str, + attempt_seconds: float = 0.0, ) -> "_MigrateAttemptBudget": """Recover from one failed `prisma migrate deploy`, and price the pass. @@ -973,37 +1127,46 @@ class ProxyExtrasDBManager: """ stderr = error.stderr or "" - if "P3005" in stderr and "database schema is not empty" in stderr: - logger.info("Schema exists but no migrations ledger — creating baseline") - if ProxyExtrasDBManager._create_baseline_migration(schema_path): - return budget.after_recovery("baseline") - return budget.spend() - if "P3009" in stderr: - migration_match = re.search(r"`(\d+_\S+?)`", stderr) - if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr): - name = migration_match.group(1) - logger.info( - f"Migration {name} failed idempotently — marking applied and retrying" - ) - ProxyExtrasDBManager._mark_migration_applied(name) - return budget.after_recovery(f"resolved:{name}") - if migration_match: - migration_name = migration_match.group(1) - ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name) - if ledger_logs is not None and ( - ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs - ): + migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr) + started_at: Final = ( + ProxyExtrasDBManager._v2_failed_migration_started_at(stderr, migration_name) if migration_name else None + ) + if migration_name and started_at: + ledger_logs: Final = ProxyExtrasDBManager._failed_migration_logs(migration_name, started_at) + if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs: logger.info( "Migration %s failed in a concurrent migrate deploy " "deadlock race, rolling its ledger row back and retrying", migration_name, ) - ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name) + ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name) + return budget.spend() + if ProxyExtrasDBManager._failed_migration_recovered(migration_name, started_at): + logger.info( + "Migration %s started at %s was already rolled back or completed by a concurrent " + "migrate deploy, retrying", + migration_name, + started_at, + ) return budget.spend() raise RuntimeError( - "Database migration failed and cannot be auto-recovered. " - f"Manual intervention required.\n\nPrisma error:\n{stderr}" + "Migration completion could not be verified. LiteLLM startup has stopped.\n\n" + f"Prisma migration history (migration name and start time):\n{stderr}\n\n" + "A migration has a start record but no successful completion record. " + "LiteLLM cannot determine whether its SQL committed from this record alone. " + "Startup stopped to avoid repeating or skipping database changes.\n\n" + "Before resolving, stop other migration runners and inspect _prisma_migrations, " + "the named migration.sql from this build, database logs, and the actual database objects and data. " + "Use the same database and this build's schema and migration files for recovery:\n" + "- Only after verifying every migration change is present, run " + "prisma migrate resolve --applied , then retry startup.\n" + "- Only after verifying no migration changes remain (or fully undoing partial changes), run " + "prisma migrate resolve --rolled-back , then retry startup. " + "This command updates history; it does not undo SQL.\n" + "Replace with the reported name. If the outcome remains uncertain, " + "leave migration history unchanged and contact your database administrator. " + "Repeated restarts alone will not resolve this state." ) from error if "P3018" in stderr: @@ -1014,25 +1177,13 @@ class ProxyExtrasDBManager: f"and retry.\n\nPrisma error:\n{stderr}" ) from error - migration_match = re.search(r"Migration name: (\d+_\S+)", stderr) - if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr): - name = migration_match.group(1) + migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr) + if migration_name and _MIGRATION_DEADLOCK_MARKER in stderr: logger.info( - f"Migration {name} SQL hit idempotent error — marking applied and retrying" - ) - ProxyExtrasDBManager._mark_migration_applied(name) - return budget.after_recovery(f"resolved:{name}") - - if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr: - logger.info( - "Migration %s deadlocked against a concurrent " - "migrate deploy, rolling its ledger row back " - "and retrying", - migration_match.group(1), - ) - ProxyExtrasDBManager._roll_back_migration_best_effort( - migration_match.group(1) + "Migration %s deadlocked against a concurrent migrate deploy, rolling its ledger row back and retrying", + migration_name, ) + ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name) return budget.spend() raise RuntimeError( @@ -1042,19 +1193,17 @@ class ProxyExtrasDBManager: if _MIGRATION_DEADLOCK_MARKER in stderr: logger.info( - "prisma migrate deploy attempt %s deadlocked against " - "a concurrent migrate deploy, retrying", + "prisma migrate deploy attempt %s deadlocked against a concurrent migrate deploy, retrying", budget.attempt_number, ) return budget.spend() if "P1002" in stderr and "advisory lock" in stderr: logger.info( - "prisma migrate deploy attempt %s timed out waiting for " - "the advisory lock a concurrent migrate deploy holds, retrying", - budget.attempt_number, + "Waiting for the advisory lock held by another Prisma migration; " + "contention does not spend a migration failure attempt" ) - return budget.spend() + return budget.after_contention(attempt_seconds) raise RuntimeError( "Database migration failed and cannot be auto-recovered. " @@ -1118,13 +1267,16 @@ class ProxyExtrasDBManager: ) @staticmethod - def setup_database( - use_migrate: bool = False, use_v2_resolver: bool = False - ) -> bool: + def setup_database(use_migrate: bool = False, use_v2_resolver: bool = False) -> bool: """ Set up the database using either prisma migrate or prisma db push Uses migrations from litellm-proxy-extras package + The request-log indexes in `REQUEST_LOG_INDEXES` are not built here: the + migration job builds them through `run_migration_job`, and a serving proxy that + ran the migrations itself starts them through `start_request_log_index_build` + once it is ready to serve. + Args: use_migrate: Whether to use prisma migrate instead of db push use_v2_resolver: Opt into the v2 migration resolver (safer during @@ -1141,10 +1293,48 @@ class ProxyExtrasDBManager: migrated = ProxyExtrasDBManager._run_migrations( use_migrate=use_migrate, use_v2_resolver=use_v2_resolver ) - if migrated: - ProxyExtrasDBManager.repair_invalid_indexes() - ProxyExtrasDBManager.apply_replica_identity_full_if_requested() - return migrated + if not migrated: + return False + ProxyExtrasDBManager.repair_invalid_indexes(skip_request_log_indexes=True) + ProxyExtrasDBManager.apply_replica_identity_full_if_requested() + return True + + @staticmethod + def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool: + """Build the indexes in `REQUEST_LOG_INDEXES` on the writer, in the schema the + migrations target. Idempotent and never raises; False when an index is still + missing or invalid, so the migration job reports it and gets rerun instead of + leaving the table unindexed until the next deploy.""" + database_url: Final = os.environ.get("DATABASE_URL") + if not database_url: + return True + direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params( + os.environ.get("DIRECT_URL") or database_url + ) + schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" + return build(direct_url, schema) + + @staticmethod + def run_migration_job( + use_migrate: bool = False, + use_v2_resolver: bool = False, + setup: Callable[[bool, bool], bool] = setup_database, + build: Callable[[], bool] = build_request_log_indexes, + ) -> bool: + """The migration job's whole run: `setup_database`, then the request-log indexes, + built synchronously so the job exits only once they are in place. False when the + migrations failed or an index could not be built, so the Job is rerun.""" + return setup(use_migrate, use_v2_resolver) and build() + + @staticmethod + def start_request_log_index_build(build: Callable[[], bool] = build_request_log_indexes) -> threading.Thread: + """A serving proxy that ran the migrations itself (schema updates not disabled) + builds the request-log indexes on a daemon thread, so a long build never delays + readiness. A build that could not finish is logged and picked up by the next boot + or the migration job.""" + thread: Final = threading.Thread(target=build, name="litellm-request-log-indexes", daemon=True) + thread.start() + return thread @staticmethod def _run_migrations(use_migrate: bool, use_v2_resolver: bool) -> bool: @@ -1188,15 +1378,16 @@ class ProxyExtrasDBManager: logger.info("✅ Post-migration sanity check completed") return True except subprocess.CalledProcessError as e: - logger.info(f"prisma db error: {e.stderr}, e: {e.stdout}") - if "P3009" in e.stderr: + stderr: Final = str(e.stderr or "") + logger.info(f"prisma db error: {stderr}, e: {e.stdout}") + if "P3009" in stderr: # Extract the failed migration name from the error message migration_match = re.search( - r"`(\d+_.*)` migration", e.stderr + r"`(\d+_.*)` migration", stderr ) if migration_match: failed_migration = migration_match.group(1) - if ProxyExtrasDBManager._is_idempotent_error(e.stderr): + if ProxyExtrasDBManager._is_idempotent_error(stderr): logger.info( f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied" ) @@ -1252,8 +1443,8 @@ class ProxyExtrasDBManager: f"✅ Migration {failed_migration} marked as rolled back... retrying" ) elif ( - "P3005" in e.stderr - and "database schema is not empty" in e.stderr + "P3005" in stderr + and "database schema is not empty" in stderr ): logger.info( "Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system" @@ -1267,13 +1458,13 @@ class ProxyExtrasDBManager: ) logger.info("✅ All migrations resolved.") return True - elif "P3018" in e.stderr: + elif "P3018" in stderr: # Check if this is a permission error or idempotent error - if ProxyExtrasDBManager._is_permission_error(e.stderr): + if ProxyExtrasDBManager._is_permission_error(stderr): # Permission errors should NOT be marked as applied # Extract migration name for logging migration_match = re.search( - r"Migration name: (\d+_.*)", e.stderr + r"Migration name: (\d+_.*)", stderr ) migration_name = ( migration_match.group(1) @@ -1283,7 +1474,7 @@ class ProxyExtrasDBManager: logger.error( f"❌ Migration {migration_name} failed due to insufficient permissions. " - f"Please check database user privileges. Error: {e.stderr}" + f"Please check database user privileges. Error: {stderr}" ) # Mark as rolled back and exit with error @@ -1306,7 +1497,7 @@ class ProxyExtrasDBManager: f"was NOT applied. Please grant necessary database permissions and retry." ) from e - elif ProxyExtrasDBManager._is_idempotent_error(e.stderr): + elif ProxyExtrasDBManager._is_idempotent_error(stderr): # Idempotent errors mean the migration has effectively been applied logger.info( "Migration failed due to idempotent error (e.g., column already exists), " @@ -1314,7 +1505,7 @@ class ProxyExtrasDBManager: ) # Extract the migration name from the error message migration_match = re.search( - r"Migration name: (\d+_.*)", e.stderr + r"Migration name: (\d+_.*)", stderr ) if migration_match: migration_name = migration_match.group(1) @@ -1363,7 +1554,7 @@ class ProxyExtrasDBManager: logger.warning( f"P3018 error encountered but could not classify " f"as permission or idempotent error. " - f"Error: {e.stderr}" + f"Error: {stderr}" ) raise else: diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 9e77f499627..9297335cd00 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.100.post1" +version = "0.4.100.post2" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.100.post1" +version = "0.4.100.post2" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py index 040d67d25e4..3917060a8dc 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -32,9 +32,7 @@ def _fake_migrate_deploy_failure(returncode: int, stderr: str): def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path): """v2: a permission failure during migrate deploy raises RuntimeError.""" monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) + monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None) monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) (tmp_path / "schema.prisma").write_text("// stub") @@ -50,9 +48,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path): def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path): """v2: a non-idempotent migration failure raises (no silent recovery).""" monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) + monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None) monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) (tmp_path / "schema.prisma").write_text("// stub") @@ -61,7 +57,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path): 'Reason: syntax error at or near "BRKN" LINE 42' ) with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -176,51 +172,33 @@ def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path): ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) -def test_v2_resolve_specific_migration_failure_raises_runtime_error( - monkeypatch, tmp_path -): - """If marking a migration as applied fails inside P3009 idempotent - recovery, the subprocess error must be re-raised as RuntimeError so - proxy_cli.py catches it cleanly (instead of leaking CalledProcessError).""" +def test_v2_duplicate_object_p3009_is_not_marked_applied(monkeypatch, tmp_path): + _stub_v2_env(monkeypatch, tmp_path) + monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name, started_at: "relation already exists") monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ProxyExtrasDBManager, + "_v2_roll_back_migration_best_effort", + lambda name: pytest.fail("duplicate-object errors do not prove rollback is safe"), ) - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") monkeypatch.setattr( - ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None + ProxyExtrasDBManager, + "_resolve_specific_migration", + lambda name: pytest.fail("duplicate-object errors do not prove all SQL completed"), ) - - # First call: migrate deploy -> P3009 idempotent error. - # Recovery path tries _resolve_specific_migration; that also raises. - def _failing_resolve(*a, **kw): - raise subprocess.CalledProcessError( - returncode=1, - cmd="prisma migrate resolve --applied", - stderr="resolve failed", - output="", - ) - - monkeypatch.setattr( - ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve - ) - - stderr = ( - "Error: P3009\nMigration `20260101000000_some_migration` failed\n" - "relation already exists" - ) - with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises( - RuntimeError, match="Failed to mark migration .* as applied" - ): + stderr = "Error: P3009\nMigration `20260101000000_some_migration` failed\nrelation already exists" + with patch( + "litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr) + ) as run: + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == ( + ["migrate", "deploy"], + ) def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): """v2 must never call _resolve_all_migrations — that's the bug it fixes.""" - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) + monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None) monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) (tmp_path / "schema.prisma").write_text("// stub") @@ -252,9 +230,7 @@ _DEADLOCK_P3018_STDERR = ( def _stub_v2_env(monkeypatch, tmp_path): monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) + monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None) monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) (tmp_path / "schema.prisma").write_text("// stub") monkeypatch.setattr("time.sleep", lambda _: None) @@ -272,9 +248,7 @@ def _succeed_after(failures: int, stderr: str): return _OkResult() calls["n"] += 1 if calls["n"] <= failures: - raise subprocess.CalledProcessError( - returncode=1, cmd=args[0], stderr=stderr, output="" - ) + raise subprocess.CalledProcessError(returncode=1, cmd=args[0], stderr=stderr, output="") return _OkResult() return _run @@ -288,7 +262,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path): rolled_back = [] monkeypatch.setattr( ProxyExtrasDBManager, - "_roll_back_migration", + "_v2_roll_back_migration_best_effort", lambda name: rolled_back.append(name), ) monkeypatch.setattr( @@ -306,7 +280,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path): def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path): """v2: a deadlock on every attempt still fails after the retry budget.""" _stub_v2_env(monkeypatch, tmp_path) - monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None) + monkeypatch.setattr(ProxyExtrasDBManager, "_v2_roll_back_migration_best_effort", lambda name: None) with patch( "litellm_proxy_extras.prisma_toolchain.run_prisma", @@ -330,12 +304,12 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_ monkeypatch.setattr( ProxyExtrasDBManager, "_failed_migration_logs", - lambda name: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock", + lambda name, started_at: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock", ) rolled_back = [] monkeypatch.setattr( ProxyExtrasDBManager, - "_roll_back_migration", + "_v2_roll_back_migration_best_effort", lambda name: rolled_back.append(name), ) monkeypatch.setattr( @@ -350,33 +324,28 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_ assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] -def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path): - """v2: empty failed ledger logs mean a concurrent deploy moved it on.""" +def test_v2_p3009_empty_ledger_logs_do_not_prove_completion(monkeypatch, tmp_path): _stub_v2_env(monkeypatch, tmp_path) - stderr = ( "Error: P3009\n" "migrate found failed migrations in the target database\n" "The `20260415120000_health_check_latest_per_model_index` migration " "started at 2026-09-01 18:46:13 UTC failed" ) - monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: "") - rolled_back = [] + monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name, started_at: "") monkeypatch.setattr( ProxyExtrasDBManager, - "_roll_back_migration", - lambda name: rolled_back.append(name), + "_v2_roll_back_migration_best_effort", + lambda name: pytest.fail("empty logs do not prove rollback is safe"), ) - monkeypatch.setattr( - ProxyExtrasDBManager, - "_resolve_specific_migration", - lambda name: pytest.fail("a deadlocked migration must never be marked applied"), + with patch( + "litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr) + ) as run: + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == ( + ["migrate", "deploy"], ) - monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr)) - - ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - assert ok is True - assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path): @@ -389,15 +358,15 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path): "The `20260415120000_health_check_latest_per_model_index` migration " "started at 2026-09-01 18:46:13 UTC failed" ) - monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: None) + monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name, started_at: None) monkeypatch.setattr( ProxyExtrasDBManager, - "_roll_back_migration", + "_v2_roll_back_migration_best_effort", lambda name: pytest.fail("an unreadable ledger must not trigger a retry"), ) monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr)) - with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -414,11 +383,11 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path): monkeypatch.setattr( ProxyExtrasDBManager, "_failed_migration_logs", - lambda name: 'ERROR: syntax error at or near "BRKN"', + lambda name, started_at: 'ERROR: syntax error at or near "BRKN"', ) with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index acd01b0e99e..98ed973db83 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -978,6 +978,28 @@ class PrismaManager: os.chdir(original_dir) return False + @staticmethod + def build_request_log_indexes() -> bool: + """Build request-log indexes after migrations and report whether all were built.""" + try: + from litellm_proxy_extras.utils import ProxyExtrasDBManager + except ImportError as e: + verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e) + return False + return ProxyExtrasDBManager.build_request_log_indexes() + + @staticmethod + def start_request_log_index_build() -> None: + """Build the request-log indexes on a daemon thread, for a serving proxy that ran the + migrations itself (`DISABLE_SCHEMA_UPDATE` unset), so a long build never delays + readiness. A build that could not finish is logged and retried on the next boot.""" + try: + from litellm_proxy_extras.utils import ProxyExtrasDBManager + except ImportError as e: + verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e) + return + ProxyExtrasDBManager.start_request_log_index_build() + def should_update_prisma_schema( disable_updates: bool | str | None = None, diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0ebe8dbc42a..834ea52aa04 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -7,9 +7,10 @@ import re import subprocess import sys import urllib.parse as urlparse -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol import click import httpx @@ -668,6 +669,33 @@ class ProxyInitializationHelpers: return multiproc_dir +class _DatabaseSetup(Protocol): + def __call__(self, *, use_migrate: bool, use_v2_resolver: bool) -> bool: ... + + +@dataclass(frozen=True, slots=True) +class _DatabaseStartupDependencies: + setup: _DatabaseSetup + build: Callable[[], bool] + start: Callable[[], None] + + +def _setup_database_with_request_log_indexes( + skip_server_startup: bool, + *, + setup: Callable[[], bool], + build: Callable[[], bool], + start: Callable[[], None], +) -> bool: + if not setup(): + return False + if skip_server_startup: + build() + return True + start() + return True + + @click.command() @click.argument("cli_args", nargs=-1) @click.option("--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST") @@ -1360,9 +1388,21 @@ def run_server( "diff-and-force recovery that caused the thrash).\033[0m" ) try: - setup_ok: Final = PrismaManager.setup_database( - use_migrate=not use_prisma_db_push, - use_v2_resolver=use_v2_migration_resolver, + dependencies: Final = click.get_current_context().find_object(_DatabaseStartupDependencies) or ( + _DatabaseStartupDependencies( + setup=PrismaManager.setup_database, + build=PrismaManager.build_request_log_indexes, + start=PrismaManager.start_request_log_index_build, + ) + ) + setup_ok: Final = _setup_database_with_request_log_indexes( + skip_server_startup, + setup=lambda: dependencies.setup( + use_migrate=not use_prisma_db_push, + use_v2_resolver=use_v2_migration_resolver, + ), + build=dependencies.build, + start=dependencies.start, ) except RuntimeError as e: # Raised on unrecoverable migration errors: the v2 diff --git a/migrations/run.py b/migrations/run.py index 7ea80d48719..94e7cc7c0f0 100644 --- a/migrations/run.py +++ b/migrations/run.py @@ -2,7 +2,12 @@ Runs `prisma migrate deploy` against the LiteLLM writer database using the recovery logic in `litellm_proxy_extras.ProxyExtrasDBManager.setup_database` -(P3005 baseline + P3009/P3018 idempotent-error handling, retries, etc.). +(P3005 baseline + P3009/P3018 idempotent-error handling, retries, etc.), then +builds the request-log indexes the migrations leave out +(`litellm_proxy_extras.request_log_indexes`), waiting for them. The job exits +non-zero when an index could not be built so that it is rerun. A serving proxy +that runs the migrations itself builds the same indexes in the background once +it serves. Env vars: DATABASE_URL required unless it can be assembled at @@ -23,10 +28,11 @@ Env vars: import os import sys -from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm_proxy_extras._logging import logger from litellm_proxy_extras.utils import ProxyExtrasDBManager, str_to_bool +from litellm.proxy.db.db_url_settings import DatabaseURLSettings + def main() -> int: # Assemble DATABASE_URL from the discrete DATABASE_* env vars, matching @@ -52,7 +58,7 @@ def main() -> int: not use_db_push, use_v2, ) - ok = ProxyExtrasDBManager.setup_database( + ok = ProxyExtrasDBManager.run_migration_job( use_migrate=not use_db_push, use_v2_resolver=use_v2, ) diff --git a/pyproject.toml b/pyproject.toml index ea52733fb6b..7c85f583d52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ proxy = [ "mcp>=2.2.0,<3", "httpx2>=2.5.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.100.post1", + "litellm-proxy-extras==0.4.100.post2", "litellm-enterprise==0.1.69.post2", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", diff --git a/tests/code_coverage_tests/check_migrations_no_data_rewrites.py b/tests/code_coverage_tests/check_migrations_no_data_rewrites.py index d7da48ce933..5c694702c6e 100644 --- a/tests/code_coverage_tests/check_migrations_no_data_rewrites.py +++ b/tests/code_coverage_tests/check_migrations_no_data_rewrites.py @@ -7,15 +7,16 @@ anything whose cost scales with existing table size turns into downtime. A singl plus a doubled heap that plain autovacuum will not give back. What is banned is the row-rewriting DML behind that, not everything whose cost -scales that way. A non-concurrent `CREATE INDEX`, an `ALTER COLUMN ... TYPE` that is -not binary coercible, a volatile `DEFAULT` on a new column, a `CREATE TABLE ... AS -SELECT` or `SELECT ... INTO` filling a new table from an existing one, the rename -that pairs with one of those to swap a table out, and a `REFRESH MATERIALIZED VIEW` -all read the whole table and all pass. That is deliberate: a rule wide enough to -reach them fires on most ordinary migrations, and a marker everyone adds by reflex -stops carrying information. The outage this was written for was a backfill. +scales that way. A non-concurrent `CREATE INDEX` passes except on a request-log +table, where it blocks writes until the build finishes. An `ALTER COLUMN ... TYPE` +that is not binary coercible, a volatile `DEFAULT` on a new column, a `CREATE TABLE +... AS SELECT` or `SELECT ... INTO` filling a new table from an existing one, the +rename that pairs with one of those to swap a table out, and a `REFRESH MATERIALIZED +VIEW` all read the whole table and all pass. That is deliberate: a rule wide enough +to reach them fires on most ordinary migrations, and a marker everyone adds by +reflex stops carrying information. The outage this was written for was a backfill. -The one schema change banned outright is `ADD COLUMN ... DEFAULT` on a table in +One column change banned outright is `ADD COLUMN ... DEFAULT` on a table in `REQUEST_LOG_TABLES`, the tables that hold a row per request. Postgres 11 stores such a default as metadata and touches no rows, but Postgres 10, which is supported, rewrites the whole heap and rebuilds every index under an `ACCESS EXCLUSIVE` lock, @@ -23,6 +24,13 @@ which on a spend-log-sized table is the same outage as a backfill. Every other t is small enough that the rewrite is not worth a rule, and a column added to a log table without a default is still free on every version. +An index on a request-log table cannot ship as a migration at all. A plain `CREATE +INDEX` blocks writes to the table until the build finishes, and `CREATE INDEX +CONCURRENTLY` is refused by Postgres on a partitioned parent, which LiteLLM_SpendLogs +is wherever the operator ran db_scripts/partition_spend_logs.sql. The migration job builds +those indexes after `migrate deploy`, concurrently and per partition, from the list in +litellm_proxy_extras/request_log_indexes.py, so that list is where a new one goes. + Flagged, per statement, by its leading keyword: UPDATE rewrites every matching row, and `WHERE` does not bound the scan @@ -44,6 +52,8 @@ Flagged, per statement, by its leading keyword: actions adds a column with a `DEFAULT`. An `ALTER COLUMN ... SET DEFAULT` written after the column exists changes metadata alone, so it passes, as does an `ADD CONSTRAINT` + CREATE only a `CREATE [UNIQUE] INDEX` on a request-log table, concurrent or + not; the migration job builds those Referential actions (`ON DELETE CASCADE`, `ON UPDATE CASCADE`) are schema, never a statement's leading keyword, so they pass. @@ -85,7 +95,9 @@ below line up with the statements they exempt. Add a column and let the application populate it, or run the rewrite as an opt-in batched job outside boot. When a rewrite is genuinely bounded and must ship inside the migration, put `-- data-migration-ok: ` on the statement or on the line -above it, naming what bounds it. The reason is required. A marker sharing a line +above it, naming what bounds it. The reason is required. A marker never exempts a +`CREATE INDEX` on a request-log table, since no bound makes that statement safe: +the migration job is the only place such an index is built. A marker sharing a line with the statement it follows exempts that statement alone, so the next statement down is still checked rather than picking the marker up as its own. A marker on an `EXECUTE` or on the assignment feeding one covers the single-quoted SQL that @@ -108,6 +120,7 @@ import sys from collections.abc import Iterator, Mapping from dataclasses import dataclass from pathlib import Path +from typing import Final REPO_ROOT = Path(__file__).resolve().parents[2] MIGRATIONS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations" @@ -115,6 +128,9 @@ MIGRATIONS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / " GRANDFATHERED = frozenset( { "20250425182129_add_session_id", + "20250510142544_add_session_id_index_spend_logs", + "20260228100000_add_spend_logs_composite_index", + "20250326162113_baseline", "20260817000000_shadow_eval_multi_key", "20260818000000_add_spend_log_timestamps", "20260818224500_add_shadow_eval_stopped_by", @@ -139,9 +155,7 @@ WORD_OR_ASSIGN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|:=|(?!:=])=(?![=>])") PRECEDING_WORD = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)[^A-Za-z0-9_]*$") QUALIFIER_GAP = re.compile(r"[\s.]*") EXPLAIN_OPTIONS = re.compile(r"\bEXPLAIN\b(?:\s+(?:ANALYZE|ANALYSE|VERBOSE)\b)+", re.IGNORECASE) -DEFINES_A_ROUTINE = re.compile( - r"\bCREATE\b(?:\s+OR\s+REPLACE)?\s+(?:FUNCTION|PROCEDURE)\b", re.IGNORECASE -) +DEFINES_A_ROUTINE = re.compile(r"\bCREATE\b(?:\s+OR\s+REPLACE)?\s+(?:FUNCTION|PROCEDURE)\b", re.IGNORECASE) QUALIFIED_NAME = r"(?:\"[^\"]*\"|[A-Za-z_][A-Za-z0-9_$]*)" ROUTINE_NAME = re.compile(rf"\s*(?:{QUALIFIED_NAME}\s*\.\s*)?({QUALIFIED_NAME})") TABLE_NAME = ROUTINE_NAME @@ -204,6 +218,12 @@ statement with the bound spelled out: -- data-migration-ok: UPDATE ... +An index on a request-log table is not a migration, and no marker exempts one. Declare it with `@@index` in +schema.prisma and add it to REQUEST_LOG_INDEXES in +litellm_proxy_extras/request_log_indexes.py under the name Prisma derives for it; the +migration job builds it after `migrate deploy`, concurrently on a plain table and per +partition on a partitioned one, which no single migration statement can do. + On Postgres 10 an `ADD COLUMN ... DEFAULT` on a request-log table rewrites the table too. Add the column nullable with no default, then set the default in a separate `ALTER COLUMN ... SET DEFAULT`, which never touches existing rows. @@ -215,10 +235,11 @@ class Violation: migration: str line: int keyword: str + consequence: str = "rewrites existing rows at boot" def render(self) -> str: location = f"{MIGRATIONS_DIR.relative_to(REPO_ROOT)}/{self.migration}/migration.sql" - return f"{location}:{self.line}: {self.keyword} rewrites existing rows at boot" + return f"{location}:{self.line}: {self.keyword} {self.consequence}" @dataclass(frozen=True, slots=True) @@ -578,6 +599,35 @@ def rewrites_a_log_table(clause: str, region: str, base: int) -> str | None: return f"ADD COLUMN ... DEFAULT on {named.group(1)}" +def builds_a_log_index(clause: str, region: str, base: int) -> str | None: + """The keyword to report when a `CREATE INDEX` targets a request-log table, concurrent or + not: a plain build blocks writes for its whole duration, and a concurrent one fails with + P3018 on a partitioned parent, so the migration job builds those instead.""" + created: Final[re.Match[str] | None] = re.match( + r"\s*CREATE\s+(?:UNIQUE\s+)?INDEX\b(?:\s+CONCURRENTLY\b)?", clause, re.IGNORECASE + ) + if created is None: + return None + on: Final[re.Match[str] | None] = re.search(r"\bON\b(?:\s+ONLY\b)?", clause[created.end() :], re.IGNORECASE) + if on is None: + return None + named: Final[re.Match[str] | None] = TABLE_NAME.match( + region, skip_comments(region, base + created.end() + on.end()) + ) + if named is None or named.group(1).strip('"') not in REQUEST_LOG_TABLES: + return None + return f"CREATE INDEX on {named.group(1)}" + + +def consequence_of(found: str) -> str: + if found.startswith("CREATE INDEX"): + return ( + "blocks writes until the build finishes, or fails on a partitioned table; " + "add it to REQUEST_LOG_INDEXES in litellm_proxy_extras/request_log_indexes.py instead" + ) + return "rewrites existing rows at boot" + + def skip_comments(sql: str, start: int) -> int: index = start while index < len(sql): @@ -746,8 +796,7 @@ def read_markers(sql: str) -> Markers: return Markers( sql, tuple( - Marker(match.start(), match.end(), alone_on_its_line(sql, match.start())) - for match in MARKER.finditer(sql) + Marker(match.start(), match.end(), alone_on_its_line(sql, match.start())) for match in MARKER.finditer(sql) ), ) @@ -760,9 +809,7 @@ def scan(sql: str, migration: str, markers: Markers) -> Iterator[Violation]: yield from scan_region(sql, sql, migration, markers, 0) -def scan_region( - document: str, region: str, migration: str, markers: Markers, offset: int -) -> Iterator[Violation]: +def scan_region(document: str, region: str, migration: str, markers: Markers, offset: int) -> Iterator[Violation]: """Violations in one region of `document`, whose text begins at `offset`. Positions are always counted against the whole document, so a statement nested in a dollar-quoted body reports its real file line and lines up with the markers read from that file. A single-quoted @@ -790,13 +837,18 @@ def scan_region( offset + start, ) - keyword = offending_keyword(clause) - if exempt: - continue - found = keyword or rewrites_a_log_table(clause, region, base) + index = builds_a_log_index(clause, region, base) + found = ( + index if exempt else offending_keyword(clause) or rewrites_a_log_table(clause, region, base) or index + ) if found is None: continue - yield Violation(migration, line_of(document, offset + keyword_start(clause, base)), found) + yield Violation( + migration, + line_of(document, offset + keyword_start(clause, base)), + found, + consequence_of(found), + ) for body in bodies: if not runs_when_applied(masked, region, bodies, runnable, identifiers, body): diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index f0303965835..eef86cdd6f0 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -16,6 +16,10 @@ import psutil from integration._support.client import Gateway +DB_PUSH: Final = ("--use_prisma_db_push",) +MIGRATE_DEPLOY: Final = ("--use_v2_migration_resolver",) +LEGACY_MIGRATE_DEPLOY: Final = () + def in_group(process: psutil.Process, group: int) -> bool: try: @@ -62,9 +66,16 @@ def owned_proxy( config: Path | None = None, remove_environment: tuple[str, ...] = (), workers: int = 1, + database_setup: tuple[str, ...] = DB_PUSH, ) -> Iterator[Gateway]: with owned_proxy_process( - gateway, directory, overrides, config=config, remove_environment=remove_environment, workers=workers + gateway, + directory, + overrides, + config=config, + remove_environment=remove_environment, + workers=workers, + database_setup=database_setup, ) as owned: yield owned.gateway @@ -78,6 +89,7 @@ def owned_proxy_process( config: Path | None = None, remove_environment: tuple[str, ...] = (), workers: int = 1, + database_setup: tuple[str, ...] = DB_PUSH, ) -> Iterator[OwnedProxy]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) @@ -109,7 +121,7 @@ def owned_proxy_process( str(workers), "--telemetry", "False", - "--use_prisma_db_push", + *database_setup, "--enforce_prisma_migration_check", ], cwd=root, diff --git a/tests/integration/database/test_request_log_indexes_at_boot.py b/tests/integration/database/test_request_log_indexes_at_boot.py new file mode 100644 index 00000000000..36242db0d70 --- /dev/null +++ b/tests/integration/database/test_request_log_indexes_at_boot.py @@ -0,0 +1,363 @@ +import os +import shutil +import subprocess +import sys +from collections.abc import Mapping +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import psycopg +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import scratch_database +from integration._support.process import LEGACY_MIGRATE_DEPLOY, MIGRATE_DEPLOY, owned_proxy_process +from psycopg import sql +from psycopg.rows import class_row + +REPO_ROOT: Final = Path(__file__).resolve().parents[3] +PRISMA_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" +PARTITION_SCRIPT: Final = REPO_ROOT / "db_scripts" / "partition_spend_logs.sql" +SHIPPED_MIGRATIONS: Final = tuple(sorted(path.name for path in (PRISMA_DIR / "migrations").iterdir() if path.is_dir())) +API_KEY_INDEX_MIGRATION: Final = "20260823000000_add_spend_logs_api_key_starttime_index" +CALL_ID_INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index" +ORIGINAL_MIGRATION_SQL: Final = MappingProxyType( + { + API_KEY_INDEX_MIGRATION: ( + "-- CreateIndex\n" + 'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" ' + 'ON "LiteLLM_SpendLogs"("api_key", "startTime");\n' + ), + CALL_ID_INDEX_MIGRATION: ( + "-- CreateIndex\n" + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" ' + 'ON "LiteLLM_SpendLogs"("litellm_call_id");\n' + ), + } +) +MIGRATION_JOB_SECONDS: Final = 300 +INDEXES_IN_PLACE: Final = "Request-log indexes are all in place" +INDEX_BUILD_LINES: Final = ("Building index", "Attached index") +BUILD_SECONDS: Final = 60 +SPEND_LOGS_INDEXES: Final = ("LiteLLM_SpendLogs_api_key_startTime_idx", "LiteLLM_SpendLogs_litellm_call_id_idx") +POPULATED_PARTITIONS: Final = MappingProxyType( + { + "LiteLLM_SpendLogs_p2026_08": ("2026-08-01", "2026-09-01"), + "LiteLLM_SpendLogs_p2026_09": ("2026-09-01", "2026-10-01"), + } +) +DEFAULT_PARTITION: Final = "LiteLLM_SpendLogs_pdefault" +ROWS_PER_PARTITION: Final = 500 +PARTITIONED_PARENT_ERROR: Final = 'cannot create index on partitioned table "LiteLLM_SpendLogs" concurrently' + + +@dataclass(frozen=True, slots=True) +class Resolver: + """One migration resolver as the serving proxy selects it (CLI flags) and as the + migration job selects it (environment).""" + + proxy_flags: tuple[str, ...] + job_environment: Mapping[str, str] + + +V2: Final = Resolver(MIGRATE_DEPLOY, MappingProxyType({"USE_V2_MIGRATION_RESOLVER": "true"})) +LEGACY: Final = Resolver(LEGACY_MIGRATE_DEPLOY, MappingProxyType({"USE_V2_MIGRATION_RESOLVER": "false"})) +RESOLVERS: Final = pytest.mark.parametrize("resolver", (V2, LEGACY), ids=("v2", "legacy")) + + +def release_layout(directory: Path, migrations: tuple[str, ...]) -> Path: + """The Prisma layout of the release that shipped `migrations`: the two index migrations + carry the SQL they shipped with, not the inert files of this build.""" + (directory / "migrations").mkdir(parents=True) + shutil.copy(PRISMA_DIR / "schema.prisma", directory / "schema.prisma") + shutil.copy(PRISMA_DIR / "migrations" / "migration_lock.toml", directory / "migrations" / "migration_lock.toml") + for name in migrations: + shutil.copytree(PRISMA_DIR / "migrations" / name, directory / "migrations" / name) + for name, original in ORIGINAL_MIGRATION_SQL.items(): + if name in migrations: + (directory / "migrations" / name / "migration.sql").write_text(original) + return directory / "schema.prisma" + + +def migrate_deploy(database_url: str, schema: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(schema)], + capture_output=True, + text=True, + timeout=300, + env={**os.environ, "DATABASE_URL": database_url}, + ) + + +def migration_job(database_url: str, resolver: Resolver) -> subprocess.CompletedProcess[str]: + """The migrations image entrypoint, as the helm migration Job runs it.""" + return subprocess.run( + [sys.executable, "-I", str(REPO_ROOT / "migrations" / "run.py")], + capture_output=True, + text=True, + timeout=MIGRATION_JOB_SECONDS, + cwd=REPO_ROOT, + env={**os.environ, "DATABASE_URL": database_url, **resolver.job_environment}, + ) + + +def migration_cli(database_url: str, gateway: Gateway, resolver: Resolver) -> subprocess.CompletedProcess[str]: + """The proxy CLI migrates, builds the indexes and exits by the migrations result.""" + return subprocess.run( + [ + sys.executable, + "-P", + "-m", + "integration._support.proxy", + "--config", + "tests/integration/proxy_config.yaml", + *resolver.proxy_flags, + "--skip_server_startup", + "--enforce_prisma_migration_check", + ], + capture_output=True, + text=True, + timeout=MIGRATION_JOB_SECONDS, + cwd=REPO_ROOT, + env={**os.environ, "DATABASE_URL": database_url, "LITELLM_MASTER_KEY": gateway.key}, + ) + + +def deploy_schema_before_the_index_migrations(database_url: str, directory: Path) -> None: + older: Final = tuple(name for name in SHIPPED_MIGRATIONS if name < API_KEY_INDEX_MIGRATION) + deployed: Final = migrate_deploy(database_url, release_layout(directory / "older-release", older)) + assert deployed.returncode == 0, deployed.stdout + deployed.stderr + + +def deploy_the_original_index_migrations(database_url: str, directory: Path) -> subprocess.CompletedProcess[str]: + """Boot the v1.103.0 layout once: on a partitioned table its CONCURRENTLY call_id index + fails with P3018 and leaves the ledger row unfinished, on a plain table both apply.""" + return migrate_deploy(database_url, release_layout(directory / "v1.103.0", SHIPPED_MIGRATIONS)) + + +def fail_the_call_id_index_migration_like_the_shipped_release(database_url: str, directory: Path) -> None: + deployed: Final = deploy_the_original_index_migrations(database_url, directory) + assert deployed.returncode != 0, deployed.stdout + assert "P3018" in deployed.stderr and PARTITIONED_PARENT_ERROR in deployed.stderr, deployed.stderr + assert ledger(database_url)[CALL_ID_INDEX_MIGRATION] is False + + +def partition_spend_logs(database_url: str) -> None: + with psycopg.connect(database_url, autocommit=True) as connection: + connection.execute(PARTITION_SCRIPT.read_bytes()) + for partition, (start, stop) in POPULATED_PARTITIONS.items(): + add_partition(connection, partition, start, stop) + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "api_key", "startTime", "endTime") ' + "SELECT %s || n, 'acompletion', 'sk-' || (n %% 7), %s::timestamp + (n * interval '1 minute'), " + "%s::timestamp + (n * interval '1 minute') + interval '1 second' FROM generate_series(1, %s) AS n", + (partition, start, start, ROWS_PER_PARTITION), + ) + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "startTime", "endTime") ' + "SELECT 'default-' || n, 'acompletion', '2026-07-01'::timestamp + (n * interval '1 minute'), " + "'2026-07-01'::timestamp + (n * interval '1 minute') FROM generate_series(1, %s) AS n", + (ROWS_PER_PARTITION,), + ) + + +@dataclass(frozen=True, slots=True) +class _LedgerRow: + name: str + finished: bool + + +@dataclass(frozen=True, slots=True) +class _IndexRow: + index: str + valid: bool + + +@dataclass(frozen=True, slots=True) +class _OidRow: + index: str + oid: int + + +@dataclass(frozen=True, slots=True) +class _AttachedRow: + partition: str + parent_index: str + valid: bool + + +def ledger(database_url: str) -> Mapping[str, bool]: + """Every migration in the ledger that was not rolled back, mapped to whether it finished.""" + with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_LedgerRow)) as cursor: + rows: Final = cursor.execute( + 'SELECT migration_name AS name, finished_at IS NOT NULL AS finished FROM "_prisma_migrations" ' + "WHERE rolled_back_at IS NULL ORDER BY migration_name" + ).fetchall() + return MappingProxyType({row.name: row.finished for row in rows}) + + +def parent_index_validity(database_url: str) -> Mapping[str, bool]: + with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_IndexRow)) as cursor: + rows: Final = cursor.execute( + "SELECT c.relname AS index, x.indisvalid AS valid FROM pg_index x JOIN pg_class c ON c.oid = x.indexrelid " + "WHERE x.indrelid = '\"LiteLLM_SpendLogs\"'::regclass AND c.relname = ANY(%s)", + (list(SPEND_LOGS_INDEXES),), + ).fetchall() + return MappingProxyType({row.index: row.valid for row in rows}) + + +def index_oids(database_url: str) -> Mapping[str, int]: + """index name -> oid for the SpendLogs indexes on the parent or any partition; a rebuild changes the oid.""" + with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_OidRow)) as cursor: + rows: Final = cursor.execute( + "SELECT c.relname AS index, c.oid::int AS oid FROM pg_index x JOIN pg_class c ON c.oid = x.indexrelid " + "WHERE x.indrelid = '\"LiteLLM_SpendLogs\"'::regclass OR x.indrelid IN " + "(SELECT inhrelid FROM pg_inherits WHERE inhparent = '\"LiteLLM_SpendLogs\"'::regclass)" + ).fetchall() + return MappingProxyType({row.index: row.oid for row in rows}) + + +def attached_partition_indexes(database_url: str) -> frozenset[tuple[str, str, bool]]: + """(partition, parent index, child is valid) for every child index attached under a SpendLogs parent index.""" + with psycopg.connect(database_url) as connection, connection.cursor(row_factory=class_row(_AttachedRow)) as cursor: + rows: Final = cursor.execute( + "SELECT part.relname AS partition, parent_index.relname AS parent_index, child.indisvalid AS valid " + "FROM pg_inherits attached " + "JOIN pg_class parent_index ON parent_index.oid = attached.inhparent " + "JOIN pg_index child ON child.indexrelid = attached.inhrelid " + "JOIN pg_class part ON part.oid = child.indrelid " + "WHERE parent_index.relname = ANY(%s)", + (list(SPEND_LOGS_INDEXES),), + ).fetchall() + return frozenset((row.partition, row.parent_index, row.valid) for row in rows) + + +def expected_attachments(partitions: tuple[str, ...]) -> frozenset[tuple[str, str, bool]]: + return frozenset((partition, index, True) for partition, index in product(partitions, SPEND_LOGS_INDEXES)) + + +def add_partition(connection: psycopg.Connection[tuple[object, ...]], partition: str, start: str, stop: str) -> None: + connection.execute( + sql.SQL('CREATE TABLE {} PARTITION OF "LiteLLM_SpendLogs" FOR VALUES FROM ({}) TO ({})').format( + sql.Identifier(partition), sql.Literal(start), sql.Literal(stop) + ) + ) + + +def assert_ready(booted_gateway: Gateway) -> None: + readiness: Final = booted_gateway.request("GET", "/health/readiness") + assert readiness.status_code == 200, readiness.text + assert readiness.json()["db"] == "connected", readiness.text + + +def assert_both_indexes_cover_every_partition(database_url: str) -> None: + """Every populated partition's index is attached and valid, both parents are valid, and + a partition created afterwards inherits both indexes.""" + populated: Final = (*POPULATED_PARTITIONS, DEFAULT_PARTITION) + assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS} + assert attached_partition_indexes(database_url) == expected_attachments(populated) + assert parent_index_validity(database_url) == {index: True for index in SPEND_LOGS_INDEXES} + with psycopg.connect(database_url, autocommit=True) as connection: + add_partition(connection, "LiteLLM_SpendLogs_p2026_10", "2026-10-01", "2026-11-01") + assert attached_partition_indexes(database_url) == expected_attachments((*populated, "LiteLLM_SpendLogs_p2026_10")) + + +def assert_the_serving_proxy_boots_and_finds_the_indexes_in_place( + gateway: Gateway, directory: Path, database_url: str, resolver: Resolver +) -> None: + """The serving proxy applies the inert files, reports ready, and its background build + finds every index already there, so it builds nothing and the catalog is untouched.""" + oids: Final = index_oids(database_url) + with owned_proxy_process( + gateway, directory, {"DATABASE_URL": database_url}, database_setup=resolver.proxy_flags + ) as booted: + assert_ready(booted.gateway) + log: Final = eventually( + lambda: booted.log.read_text(errors="replace"), lambda text: INDEXES_IN_PLACE in text, seconds=BUILD_SECONDS + ) + assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS} + assert not any(line in log for line in INDEX_BUILD_LINES), log[-4000:] + assert index_oids(database_url) == oids + + +@RESOLVERS +def test_the_migration_job_gives_a_partitioned_table_at_the_pre_index_schema_both_indexes_per_partition( + gateway: Gateway, tmp_path: Path, resolver: Resolver +) -> None: + with scratch_database() as database_url: + deploy_schema_before_the_index_migrations(database_url, tmp_path) + partition_spend_logs(database_url) + job: Final = migration_job(database_url, resolver) + assert job.returncode == 0, job.stdout + job.stderr + assert INDEXES_IN_PLACE in job.stderr + job.stdout, job.stdout + job.stderr + assert_both_indexes_cover_every_partition(database_url) + assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(gateway, tmp_path, database_url, resolver) + + +@RESOLVERS +def test_the_migration_job_heals_a_partitioned_table_left_with_the_failed_call_id_ledger_row( + gateway: Gateway, tmp_path: Path, resolver: Resolver +) -> None: + with scratch_database() as database_url: + deploy_schema_before_the_index_migrations(database_url, tmp_path) + partition_spend_logs(database_url) + fail_the_call_id_index_migration_like_the_shipped_release(database_url, tmp_path) + job: Final = migration_job(database_url, resolver) + assert job.returncode == 0, job.stdout + job.stderr + assert_both_indexes_cover_every_partition(database_url) + assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(gateway, tmp_path, database_url, resolver) + + +@RESOLVERS +def test_the_migration_job_leaves_a_plain_table_that_applied_the_original_index_migrations_alone( + gateway: Gateway, tmp_path: Path, resolver: Resolver +) -> None: + with scratch_database() as database_url: + deploy_schema_before_the_index_migrations(database_url, tmp_path) + deployed: Final = deploy_the_original_index_migrations(database_url, tmp_path) + assert deployed.returncode == 0, deployed.stdout + deployed.stderr + before: Final = index_oids(database_url) + assert set(SPEND_LOGS_INDEXES) <= set(before), before + job: Final = migration_job(database_url, resolver) + assert job.returncode == 0, job.stdout + job.stderr + assert INDEXES_IN_PLACE in job.stderr + job.stdout, job.stdout + job.stderr + assert "Building index" not in job.stderr + job.stdout, job.stdout + job.stderr + assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS} + assert index_oids(database_url) == before + assert_the_serving_proxy_boots_and_finds_the_indexes_in_place(gateway, tmp_path, database_url, resolver) + + +@RESOLVERS +def test_a_serving_proxy_that_runs_the_migrations_itself_builds_both_indexes_after_it_is_ready( + gateway: Gateway, tmp_path: Path, resolver: Resolver +) -> None: + """A deployment that runs migrate deploy from the serving proxy and never runs the + migration job answers readiness with the inert files applied, then its background build + puts both indexes on every partition.""" + with scratch_database() as database_url: + deploy_schema_before_the_index_migrations(database_url, tmp_path) + partition_spend_logs(database_url) + assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in parent_index_validity(database_url) + with owned_proxy_process( + gateway, tmp_path, {"DATABASE_URL": database_url}, database_setup=resolver.proxy_flags + ) as booted: + assert_ready(booted.gateway) + assert ledger(database_url) == {name: True for name in SHIPPED_MIGRATIONS} + log: Final = eventually( + lambda: booted.log.read_text(errors="replace"), + lambda text: INDEXES_IN_PLACE in text, + seconds=BUILD_SECONDS, + ) + assert "Building index" in log and "Attached index" in log, log[-4000:] + assert_both_indexes_cover_every_partition(database_url) + + +def test_the_cli_run_as_a_migration_job_builds_both_indexes_before_it_exits(gateway: Gateway, tmp_path: Path) -> None: + with scratch_database() as database_url: + deploy_schema_before_the_index_migrations(database_url, tmp_path) + partition_spend_logs(database_url) + job: Final = migration_cli(database_url, gateway, V2) + assert job.returncode == 0, job.stdout + job.stderr + assert_both_indexes_cover_every_partition(database_url) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index ec994c2186c..fe1c934e680 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -2,7 +2,12 @@ import glob import os import re import sys +from collections.abc import Callable +from dataclasses import dataclass +import threading from pathlib import Path +from queue import SimpleQueue +from typing import Final import pytest @@ -13,10 +18,10 @@ sys.path.insert( ), ) +from litellm_proxy_extras.request_log_indexes import filter_request_log_index_diff from litellm_proxy_extras.utils import ( PARTITIONED_SPEND_LOGS_PUSH_ERROR, ProxyExtrasDBManager, - filter_hand_built_spend_logs_index_diff, filter_partitioned_spend_logs_diff, ) @@ -29,6 +34,122 @@ _MIGRATIONS_DIR = os.path.abspath( ) +def test_coordination_failure_does_not_misclassify_fatal_password_as_unreachable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import psycopg + from litellm_proxy_extras.migration_lock import _coordination_failure + + database_url: Final = "postgresql://u:FATAL@127.0.0.1:5432/db" + monkeypatch.setenv("DATABASE_URL", database_url) + error: Final = psycopg.OperationalError( + 'connection failed: connection to server at "127.0.0.1", port 5432 failed: ' + 'FATAL: password authentication failed for user "u"' + ) + + result: Final = _coordination_failure(database_url, error, connecting=True) + + assert result.startswith("Lost or could not establish v2 migration coordination with the database: ") + assert "P1001" not in result + + +@pytest.mark.parametrize( + ("message", "detail"), + [ + ( + 'connection failed: connection to server at "127.0.0.1", port 5432 failed: SSL error: ' + "certificate verify failed", + "certificate verify failed", + ), + ( + 'connection failed: connection to server at "127.0.0.1", port 5432 failed: server certificate ' + 'for "db.internal" does not match host name "127.0.0.1"', + "does not match host name", + ), + ( + 'connection failed: connection to server at "127.0.0.1", port 5432 failed: server does not support ' + "SSL, but SSL was required", + "SSL was required", + ), + ], +) +def test_coordination_failure_does_not_label_tls_failures_as_unreachable( + monkeypatch: pytest.MonkeyPatch, + message: str, + detail: str, +) -> None: + import psycopg + from litellm_proxy_extras.migration_lock import _coordination_failure + + database_url: Final = "postgresql://u:pw@127.0.0.1:5432/db" + monkeypatch.setenv("DATABASE_URL", database_url) + error: Final = psycopg.OperationalError(message) + + result: Final = _coordination_failure(database_url, error, connecting=True) + + assert result.startswith("Lost or could not establish v2 migration coordination with the database: ") + assert "P1001" not in result + assert detail in result + + +@pytest.mark.parametrize( + ("database_url", "message", "host"), + [ + ( + "postgresql://u:pw@certificate-db.internal:5432/db", + 'connection failed: connection to server at "certificate-db.internal" (10.0.0.5), port 5432 failed: ' + "Connection refused", + "certificate-db.internal", + ), + ( + "postgresql://u:pw@SSL-primary:5432/db", + 'connection failed: connection to server at "SSL-primary" (10.0.0.6), port 5432 failed: ' + "Connection refused", + "ssl-primary", + ), + ( + "postgresql://u:pw@certificate-db.invalid:5432/db", + "connection failed: failed to resolve host 'certificate-db.invalid': [Errno -2] " + "Name or service not known", + "certificate-db.invalid", + ), + ], +) +def test_coordination_failure_labels_unreachable_hosts_named_like_tls_markers( + monkeypatch: pytest.MonkeyPatch, + database_url: str, + message: str, + host: str, +) -> None: + import psycopg + from litellm_proxy_extras.migration_lock import _coordination_failure + + monkeypatch.setenv("DATABASE_URL", database_url) + error: Final = psycopg.OperationalError(message) + + result: Final = _coordination_failure(database_url, error, connecting=True) + + assert f"P1001: Can't reach database server at `{host}:5432`" in result + + +def test_coordination_failure_labels_connection_refused_as_unreachable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import psycopg + from litellm_proxy_extras.migration_lock import _coordination_failure + + database_url: Final = "postgresql://u:pw@127.0.0.1:5432/db" + monkeypatch.setenv("DATABASE_URL", database_url) + error: Final = psycopg.OperationalError( + 'connection failed: connection to server at "127.0.0.1", port 5432 failed: Connection refused' + ) + + result: Final = _coordination_failure(database_url, error, connecting=True) + + assert result.startswith("Lost or could not establish v2 migration coordination with the database: ") + assert "P1001: Can't reach database server at `127.0.0.1:5432`" in result + + def test_custom_prisma_dir(monkeypatch): import tempfile @@ -644,14 +765,14 @@ class TestHandBuiltSpendLogsIndexDriftFilter: ], ) def test_hand_built_index_creation_is_removed(self, statement): - assert filter_hand_built_spend_logs_index_diff(f"-- CreateIndex\n{statement};\n") == "" + assert filter_request_log_index_diff(f"-- CreateIndex\n{statement};\n") == "" def test_other_spend_logs_indexes_and_statements_are_kept_verbatim(self): kept = ( 'CREATE INDEX "LiteLLM_SpendLogs_session_id_idx" ON "LiteLLM_SpendLogs"("session_id");\n' 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;\n' ) - assert filter_hand_built_spend_logs_index_diff(kept) == kept + assert filter_request_log_index_diff(kept) == kept class TestPartitionedSpendLogsPushGuard: @@ -761,6 +882,9 @@ class TestSpendLogsPartitionDetectionMissingPsycopg: _ATTEMPT_BUDGET = 4 +_P3009_MIGRATION_NAME = "20260415120000_health_check_latest_per_model_index" +_P3009_STARTED_AT = "2026-10-02 23:20:56.439594 UTC" +_P3009_DEADLOCK_LOGS = "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock on transaction 991" _P3005_STDERR = """Error: P3005 @@ -782,12 +906,120 @@ ERROR: relation "SomeTable" already exists """ +def _p3009_stderr(migration_name: str, started_at: str) -> str: + return ( + "Error: P3009\n\n" + "migrate found failed migrations in the target database, new migrations will not be applied. " + "Read more about how to resolve migration issues in a production database: " + "https://pris.ly/d/migrate-resolve\n" + f"The `{migration_name}` migration started at {started_at} failed\n" + ) + + +@dataclass(frozen=True, slots=True) +class _LedgerRow: + migration_name: str + started_at: str + finished: bool = False + rolled_back: bool = False + logs: str | None = None + + +@dataclass(frozen=True, slots=True) +class _LedgerCursor: + row: tuple[object, ...] | None = None + + def fetchone(self) -> tuple[object, ...] | None: + return self.row + + def fetchall(self) -> tuple[tuple[object, ...], ...]: + return () + + +class _LedgerConnection: + def __init__(self, ledger: "_FakeLedger") -> None: + self.ledger = ledger + + def __enter__(self) -> "_LedgerConnection": + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute(self, query: object, params: tuple[object, ...] = ()) -> _LedgerCursor: + return self.ledger.execute(query, params) + + +class _FakeLedger: + def __init__(self, at_error: tuple[_LedgerRow, ...], after_peer: tuple[_LedgerRow, ...]) -> None: + self.rows = at_error + self.after_peer = after_peer + self._peer_observed = False + + def connect(self, *args: object, **kwargs: object) -> _LedgerConnection: + return _LedgerConnection(self) + + def execute(self, query: object, params: tuple[object, ...]) -> _LedgerCursor: + text: Final = str(query) + if "WHERE migration_name = %s" not in text or not params: + return _LedgerCursor() + if not self._peer_observed: + self.rows = self.after_peer + self._peer_observed = True + matching: Final = tuple( + row + for row in self.rows + if row.migration_name == params[0] and (len(params) == 1 or row.started_at == params[1]) + ) + if "rolled_back_at IS NULL" in text: + unresolved: Final = next((row for row in matching if not row.finished and not row.rolled_back), None) + return _LedgerCursor((unresolved.logs,) if unresolved else None) + if "IS NOT NULL" in text: + resolved: Final = next((row for row in matching if row.finished or row.rolled_back), None) + return _LedgerCursor((1,) if resolved else None) + return _LedgerCursor() + + +@pytest.mark.parametrize( + "pooled,direct,expected", + ( + ("postgresql://pool/db?pgbouncer=true", None, "postgresql://pool/db?pgbouncer=true"), + ("postgresql://pool/db?pgbouncer=true", "postgresql://writer/db", "postgresql://writer/db?schema=public"), + ( + "postgresql://pool/db?schema=tenant%20one&pgbouncer=true", + "postgresql://writer/db?sslmode=require&schema=wrong", + "postgresql://writer/db?sslmode=require&schema=tenant+one", + ), + ), +) +def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema( + pooled: str, direct: str | None, expected: str +) -> None: + from litellm_proxy_extras.migration_lock import migration_environment + + environment: Final = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"} + configured: Final = {**environment, **({"DIRECT_URL": direct} if direct else {})} + migrated: Final = migration_environment(configured) + + assert migrated["DATABASE_URL"] == expected + assert migrated["PRISMA_OFFLINE_MODE"] == "true" + assert configured["DATABASE_URL"] == pooled + + class _MigrateDeployHarness: """Drives _setup_database_v2 with a scripted sequence of `prisma migrate deploy` outcomes, with every recovery command faked out so nothing touches a database or the packaged migrations directory.""" - def __init__(self, monkeypatch, tmp_path, outcomes, repeat_last=False): + def __init__( + self, + monkeypatch, + tmp_path, + outcomes, + repeat_last=False, + confirmed_migrations=(), + ledger: "_FakeLedger | None" = None, + ): import subprocess as subprocess_module import litellm_proxy_extras.utils as utils_module @@ -798,16 +1030,9 @@ class _MigrateDeployHarness: self._outcomes = list(outcomes) self._repeat_last = repeat_last self._subprocess_module = subprocess_module + self.confirmed_migrations = set(confirmed_migrations) - monkeypatch.delenv("DATABASE_URL", raising=False) - monkeypatch.setattr( - ProxyExtrasDBManager, "_get_prisma_dir", staticmethod(lambda: str(tmp_path)) - ) - monkeypatch.setattr( - ProxyExtrasDBManager, - "_create_baseline_migration", - staticmethod(self._fake_baseline), - ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", staticmethod(lambda: str(tmp_path))) monkeypatch.setattr( ProxyExtrasDBManager, "_roll_back_migration", @@ -818,14 +1043,22 @@ class _MigrateDeployHarness: "_resolve_specific_migration", staticmethod(self.resolved.append), ) + if ledger is None: + monkeypatch.delenv("DATABASE_URL", raising=False) + else: + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr("psycopg.connect", ledger.connect) + monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path)) monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", self._fake_run) + monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: {}) monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None) self.baseline_succeeds = True def _fake_baseline(self, *args, **kwargs): self.baselines += 1 - return self.baseline_succeeds + if not self.baseline_succeeds: + raise RuntimeError("The existing schema was not verified") def _next_outcome(self): if self._outcomes: @@ -845,79 +1078,214 @@ class _MigrateDeployHarness: raise self._subprocess_module.CalledProcessError(1, cmd, stderr=outcome) def run(self): - return ProxyExtrasDBManager._setup_database_v2(use_migrate=True) + while not ProxyExtrasDBManager._run_database_v2( + use_migrate=True, + recover_completed=self._fake_recovery, + baseline_existing=self._fake_baseline, + ): + continue + return True + + def _fake_recovery(self, name: str) -> bool: + if name not in self.confirmed_migrations: + return False + self.confirmed_migrations.remove(name) + self.resolved.append(name) + return True + + +class TestConcurrentP3009Recovery: + @pytest.mark.parametrize( + "after_peer", + ( + (_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS),), + ( + _LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS), + _LedgerRow(_P3009_MIGRATION_NAME, "2026-10-02 23:21:11.539224 UTC"), + ), + (_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, finished=True, logs=_P3009_DEADLOCK_LOGS),), + ), + ids=("rolled-back", "rolled-back-beside-a-fresh-in-flight-row", "finished"), + ) + def test_a_p3009_row_a_peer_already_recovered_is_retried( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + after_peer: tuple[_LedgerRow, ...], + ) -> None: + deadlocked_row: Final = _LedgerRow( + _P3009_MIGRATION_NAME, + _P3009_STARTED_AT, + logs=_P3009_DEADLOCK_LOGS, + ) + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT), "ok"], + ledger=_FakeLedger(at_error=(deadlocked_row,), after_peer=after_peer), + ) + + assert harness.run() is True + assert len(harness.deploy_calls) == 2 + + @pytest.mark.parametrize( + "ledger_rows", + ( + ( + _LedgerRow( + _P3009_MIGRATION_NAME, + _P3009_STARTED_AT, + logs='ERROR: syntax error at or near "SLECT"', + ), + ), + ( + _LedgerRow( + _P3009_MIGRATION_NAME, + _P3009_STARTED_AT, + logs='ERROR: syntax error at or near "SLECT"', + ), + _LedgerRow( + _P3009_MIGRATION_NAME, + "2026-10-02 23:19:40.120000 UTC", + rolled_back=True, + logs=_P3009_DEADLOCK_LOGS, + ), + ), + ), + ids=("only-row", "beside-a-recovered-earlier-attempt"), + ) + def test_an_unresolved_p3009_row_without_the_deadlock_marker_stops( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ledger_rows: tuple[_LedgerRow, ...], + ) -> None: + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT)], + ledger=_FakeLedger(at_error=ledger_rows, after_peer=ledger_rows), + ) + + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): + harness.run() + assert len(harness.deploy_calls) == 1 class TestMigrateDeployAttemptAccounting: - """A `prisma db push` database has a full schema and no ledger, so the v2 - resolver baselines it and then works through every migration whose objects - already exist. Those recoveries make progress, so they must not spend the - retry budget, which is there to stop a run that is getting nowhere.""" - - def test_a_push_created_database_finishes_bootstrapping( - self, monkeypatch, tmp_path - ): - already_there = [ - "20250329084805_new_cron_job_table", - "20250806095134_rename_alias_to_server_name_mcp_table", - "20260224203854_add_agent_object_permissions_table", - "20260301120000_fourth_table", - "20260302120000_fifth_table", - "20260303120000_sixth_table", - ] + def test_a_push_created_database_finishes_bootstrapping(self, monkeypatch, tmp_path): harness = _MigrateDeployHarness( monkeypatch, tmp_path, - [_P3005_STDERR] - + [_p3018_stderr(name) for name in already_there] - + ["ok"], + [_P3005_STDERR, "ok"], ) assert harness.run() is True assert harness.baselines == 1 - assert harness.resolved == already_there - assert len(harness.deploy_calls) == len(already_there) + 2 + assert harness.resolved == [] + assert len(harness.deploy_calls) == 2 - def test_repeated_recovery_of_one_migration_still_gives_up( - self, monkeypatch, tmp_path - ): + def test_repeated_recovery_of_one_migration_still_gives_up(self, monkeypatch, tmp_path): harness = _MigrateDeployHarness( monkeypatch, tmp_path, [_p3018_stderr("20250329084805_new_cron_job_table")], repeat_last=True, + confirmed_migrations=("20250329084805_new_cron_job_table",), ) with pytest.raises(RuntimeError): harness.run() - assert len(harness.deploy_calls) <= _ATTEMPT_BUDGET + 1 + assert len(harness.deploy_calls) == 2 + assert harness.resolved == ["20250329084805_new_cron_job_table"] def test_timeouts_still_spend_the_budget(self, monkeypatch, tmp_path): - harness = _MigrateDeployHarness( - monkeypatch, tmp_path, ["timeout"], repeat_last=True - ) + harness = _MigrateDeployHarness(monkeypatch, tmp_path, ["timeout"], repeat_last=True) with pytest.raises(RuntimeError): harness.run() assert len(harness.deploy_calls) == _ATTEMPT_BUDGET - def test_a_baseline_that_never_lands_stops_after_the_budget( - self, monkeypatch, tmp_path - ): - harness = _MigrateDeployHarness( - monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True - ) + def test_an_unverified_baseline_stops_without_replaying_migrations( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + harness: Final = _MigrateDeployHarness(monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True) harness.baseline_succeeds = False with pytest.raises(RuntimeError): harness.run() - assert len(harness.deploy_calls) == _ATTEMPT_BUDGET + assert len(harness.deploy_calls) == 1 + + def test_lock_contention_does_not_spend_the_failure_budget( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + ["Error: P1002\nTimed out waiting for the advisory lock"] * 6 + ["ok"], + ) + assert harness.run() is True + assert len(harness.deploy_calls) == 7 + + def test_duplicate_object_error_without_completion_proof_is_fatal( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + harness: Final = _MigrateDeployHarness(monkeypatch, tmp_path, [_p3018_stderr("20260101000000_x")]) + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + harness.run() + assert harness.resolved == [] + assert len(harness.deploy_calls) == 1 + + @pytest.mark.parametrize("name", ("20260101000000_x", "20260101000000_migration with spaces")) + def test_an_interrupted_migration_with_confirmed_sql_can_finish( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, name: str + ) -> None: + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [f"Error: P3009\nThe `{name}` migration failed", "ok"], + confirmed_migrations=(name,), + ) + assert harness.run() is True + assert harness.resolved == [name] + + def test_an_interrupted_migration_without_confirmation_stops( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + name: Final = "20260101000000_x" + started: Final = "2026-09-12 20:15:06.694553 UTC" + report: Final = f"Error: P3009\nThe `{name}` migration started at {started} failed" + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [report], + ) + with pytest.raises(RuntimeError, match="Migration completion could not be verified") as failure: + harness.run() + message: Final = str(failure.value) + assert name in message + assert started in message + assert "start record but no successful completion record" in message + assert "cannot determine whether its SQL committed" in message + assert "avoid repeating or skipping database changes" in message + assert "_prisma_migrations" in message + assert "migration.sql" in message + assert "same database" in message + assert "Only after verifying every migration change is present" in message + assert "prisma migrate resolve --applied " in message + assert "Only after verifying no migration changes remain" in message + assert "prisma migrate resolve --rolled-back " in message + assert "leave migration history unchanged" in message + assert "Repeated restarts alone" in message + assert report in message + assert len(harness.deploy_calls) == 1 + assert harness.resolved == [] def test_an_unrecoverable_error_is_not_retried(self, monkeypatch, tmp_path): harness = _MigrateDeployHarness( monkeypatch, tmp_path, - ["Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near \"SLECT\"\n"], + ['Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near "SLECT"\n'], repeat_last=True, ) @@ -927,6 +1295,38 @@ class TestMigrateDeployAttemptAccounting: assert harness.resolved == [] +@pytest.mark.parametrize( + "steps,logs,script,expected", + ( + (1, "", b"CREATE TABLE item (id int);", True), + (0, "", b"CREATE TABLE item (id int);", False), + (0, "already exists", b"CREATE TABLE item (id int);", False), + (1, "permission denied", b"CREATE TABLE item (id int);", False), + (1, "", b"CREATE TABLE item (id text);", False), + (2, "", b"CREATE TABLE item (id int);", False), + ), +) +def test_migration_completion_requires_a_matching_successful_script( + steps: int, logs: str, script: bytes, expected: bool +) -> None: + import hashlib + + from litellm_proxy_extras.migration_recovery import MigrationProgress + + progress: Final = MigrationProgress(hashlib.sha256(b"CREATE TABLE item (id int);").hexdigest(), steps, logs) + assert progress.confirms_completion(script) is expected + + +def test_prisma_lock_waiting_has_its_own_deadline(): + from litellm_proxy_extras.utils import _MigrateAttemptBudget + + budget = _MigrateAttemptBudget(attempts_left=4, contention_seconds_left=2) + waiting = budget.after_contention(1) + assert waiting.attempts_left == 4 + with pytest.raises(RuntimeError, match="advisory lock"): + waiting.after_contention(2) + + class TestJWTKeyMappingCascade: """Regression tests for issue #33702. @@ -991,3 +1391,329 @@ class TestJWTKeyMappingCascade: f"{path} must declare onDelete: Cascade on the JWT key mapping " "relation (issue #33702)" ) + + +class TestStripPrismaQueryParams: + """The psycopg URL the job connects with is derived from the Prisma-dialect + DATABASE_URL, whose TLS params mean something else to libpq.""" + + @staticmethod + def _query(url: str) -> dict[str, str]: + from urllib.parse import parse_qsl, urlparse + + return dict(parse_qsl(urlparse(url).query)) + + @pytest.mark.parametrize( + "url,expected_user,expected_password,expected_host,expected_port,expected_sslmode", + ( + ( + "postgresql://u:wr ong'pw9@writer:5433/litellm", + "u", + "wr ong'pw9", + "writer", + "5433", + None, + ), + ( + "postgresql://u:wr ong'pw9@writer:5433/litellm?schema=x&sslmode=require", + "u", + "wr ong'pw9", + "writer", + "5433", + "require", + ), + ( + "postgresql://u:p%40ss%20w@writer:5433/litellm", + "u", + "p@ss w", + "writer", + "5433", + None, + ), + ( + "postgresql://u:p%z@writer:5433/litellm", + "u", + "p%z", + "writer", + "5433", + None, + ), + ( + "postgresql://u:wr ong'pw9@[::1]:5433/litellm", + "u", + "wr ong'pw9", + "::1", + "5433", + None, + ), + ), + ) + def test_userinfo_is_encoded_for_libpq( + self, + url: str, + expected_user: str, + expected_password: str, + expected_host: str, + expected_port: str, + expected_sslmode: str | None, + ) -> None: + from psycopg.conninfo import conninfo_to_dict + + result: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + conninfo: Final = conninfo_to_dict(result) + + assert conninfo["user"] == expected_user + assert conninfo["password"] == expected_password + assert conninfo["host"] == expected_host + assert conninfo["port"] == expected_port + assert conninfo["dbname"] == "litellm" + assert conninfo.get("sslmode") == expected_sslmode + assert "schema" not in conninfo + + def test_empty_password_userinfo_keeps_its_separator(self) -> None: + url: Final = "postgresql://u:@writer/litellm" + + assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url + + def test_prisma_ca_sslcert_becomes_sslrootcert_with_verify_full(self) -> None: + url: Final = ( + "postgresql://u:p@writer:5432/db?schema=public&sslmode=require&sslcert=/tmp/pinned.pem&sslaccept=strict" + ) + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/tmp/pinned.pem"} + assert cleaned.startswith("postgresql://u:p@writer:5432/db?") + + @pytest.mark.parametrize("sslmode", ["prefer", "require"]) + @pytest.mark.parametrize("sslaccept", ["strict", "unknown-mode-prisma-treats-as-strict"]) + def test_strict_verifies_chain_and_hostname_whatever_sslmode_prisma_was_given( + self, sslmode: str, sslaccept: str + ) -> None: + url: Final = f"postgresql://writer/db?sslmode={sslmode}&sslcert=/certs/ca.pem&sslaccept={sslaccept}" + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/certs/ca.pem"} + + def test_strict_with_tls_disabled_stays_off(self) -> None: + url: Final = "postgresql://writer/db?sslmode=disable&sslcert=/certs/ca.pem&sslaccept=strict" + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert self._query(cleaned) == {"sslmode": "disable"} + + @pytest.mark.parametrize("sslaccept", ["&sslaccept=accept_invalid_certs", ""]) + def test_without_strict_the_ca_is_dropped_so_libpq_checks_nothing_like_prisma(self, sslaccept: str) -> None: + url: Final = f"postgresql://writer/db?sslmode=require&sslcert=/certs/ca.pem{sslaccept}" + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert self._query(cleaned) == {"sslmode": "require"} + + def test_a_ca_alone_without_strict_or_sslmode_leaves_libpq_its_defaults(self) -> None: + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params("postgresql://writer/db?sslcert=/certs/ca.pem") + + assert cleaned == "postgresql://writer/db" + + def test_a_libpq_client_certificate_pair_is_left_alone(self) -> None: + url: Final = ( + "postgresql://writer/db?sslmode=verify-full&sslrootcert=/ca.pem&sslcert=/client.crt&sslkey=/client.key" + ) + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert self._query(cleaned) == { + "sslmode": "verify-full", + "sslrootcert": "/ca.pem", + "sslcert": "/client.crt", + "sslkey": "/client.key", + } + + def test_an_explicit_sslrootcert_wins_over_the_prisma_sslcert(self) -> None: + url: Final = "postgresql://writer/db?sslmode=require&sslrootcert=/ca.pem&sslcert=/pinned.pem&sslaccept=strict" + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/ca.pem"} + + def test_prisma_only_params_are_dropped_and_plain_urls_pass_through(self) -> None: + url: Final = "postgresql://u:p@pooler:6543/db?schema=tenant&pgbouncer=true&connection_limit=5&connect_timeout=3" + + cleaned: Final = ProxyExtrasDBManager._strip_prisma_query_params(url) + + assert cleaned == "postgresql://u:p@pooler:6543/db?connect_timeout=3" + assert ( + ProxyExtrasDBManager._strip_prisma_query_params("postgresql://u:p@writer/db") + == "postgresql://u:p@writer/db" + ) + + +class TestBuildRequestLogIndexes: + """The migration job hands the index build the direct database URL and the schema + the migrations target, waits for it, and reports its result.""" + + @pytest.fixture + def builds(self) -> SimpleQueue[tuple[str, str]]: + return SimpleQueue() + + @pytest.fixture + def build(self, builds: SimpleQueue[tuple[str, str]]) -> Callable[[str, str], bool]: + def record(database_url: str, schema: str) -> bool: + builds.put((database_url, schema)) + return True + + return record + + def test_the_build_gets_the_direct_url_without_prisma_params_and_the_prisma_schema( + self, monkeypatch: pytest.MonkeyPatch, builds: SimpleQueue[tuple[str, str]], build: Callable[[str, str], bool] + ) -> None: + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@pooler:6543/db?schema=tenant&pgbouncer=true") + monkeypatch.setenv("DIRECT_URL", "postgresql://u:p@primary:5432/db?connection_limit=1") + + assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True + + assert builds.get_nowait() == ("postgresql://u:p@primary:5432/db", "tenant") + assert builds.empty() + + def test_the_build_defaults_to_the_database_url_and_the_public_schema( + self, monkeypatch: pytest.MonkeyPatch, builds: SimpleQueue[tuple[str, str]], build: Callable[[str, str], bool] + ) -> None: + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@primary:5432/db") + monkeypatch.delenv("DIRECT_URL", raising=False) + + assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True + + assert builds.get_nowait() == ("postgresql://u:p@primary:5432/db", "public") + assert builds.empty() + + def test_a_build_that_leaves_indexes_missing_is_reported_so_the_job_reruns( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@primary:5432/db") + + assert ProxyExtrasDBManager.build_request_log_indexes(build=lambda url, schema: False) is False + + def test_without_a_database_url_nothing_is_built( + self, monkeypatch: pytest.MonkeyPatch, builds: SimpleQueue[tuple[str, str]], build: Callable[[str, str], bool] + ) -> None: + monkeypatch.delenv("DATABASE_URL", raising=False) + + assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True + + assert builds.empty() + + +class TestStartRequestLogIndexBuild: + """A serving proxy that ran the migrations starts the index build on a daemon thread + and goes on to serve while it runs.""" + + def test_the_build_runs_on_a_daemon_thread_that_does_not_hold_up_the_caller(self) -> None: + release: Final = threading.Event() + builds: Final[SimpleQueue[str]] = SimpleQueue() + + def build() -> bool: + assert release.wait(5), "the caller never came back from start_request_log_index_build" + builds.put(threading.current_thread().name) + return True + + thread: Final = ProxyExtrasDBManager.start_request_log_index_build(build=build) + + assert builds.empty(), "the build ran before start_request_log_index_build returned" + assert thread.daemon is True + release.set() + thread.join(5) + assert builds.get_nowait() == "litellm-request-log-indexes" + assert builds.empty() + + +class TestRunMigrationJob: + """`run_migration_job` is `setup_database` followed by the index build, each step's + result deciding whether the job reports success.""" + + @pytest.fixture + def calls(self) -> SimpleQueue[tuple[str, bool, bool] | tuple[str]]: + return SimpleQueue() + + @pytest.fixture + def setup( + self, calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]] + ) -> Callable[[bool], Callable[[bool, bool], bool]]: + def record(result: bool) -> Callable[[bool, bool], bool]: + def setup_database(use_migrate: bool, use_v2_resolver: bool) -> bool: + calls.put(("setup", use_migrate, use_v2_resolver)) + return result + + return setup_database + + return record + + @pytest.fixture + def build(self, calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]]) -> Callable[[bool], Callable[[], bool]]: + def record(result: bool) -> Callable[[], bool]: + def build_request_log_indexes() -> bool: + calls.put(("build",)) + return result + + return build_request_log_indexes + + return record + + def test_the_job_builds_the_indexes_after_the_migrations_succeed( + self, + calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]], + setup: Callable[[bool], Callable[[bool, bool], bool]], + build: Callable[[bool], Callable[[], bool]], + ) -> None: + assert ProxyExtrasDBManager.run_migration_job(True, False, setup=setup(True), build=build(True)) is True + + assert (calls.get_nowait(), calls.get_nowait()) == (("setup", True, False), ("build",)) + assert calls.empty() + + def test_the_job_fails_without_building_when_the_migrations_fail( + self, + calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]], + setup: Callable[[bool], Callable[[bool, bool], bool]], + build: Callable[[bool], Callable[[], bool]], + ) -> None: + assert ProxyExtrasDBManager.run_migration_job(True, True, setup=setup(False), build=build(True)) is False + + assert calls.get_nowait() == ("setup", True, True) + assert calls.empty() + + def test_the_job_fails_when_an_index_could_not_be_built( + self, + calls: SimpleQueue[tuple[str, bool, bool] | tuple[str]], + setup: Callable[[bool], Callable[[bool, bool], bool]], + build: Callable[[bool], Callable[[], bool]], + ) -> None: + assert ProxyExtrasDBManager.run_migration_job(True, True, setup=setup(True), build=build(False)) is False + + assert (calls.get_nowait(), calls.get_nowait()) == (("setup", True, True), ("build",)) + assert calls.empty() + + +class TestMigrationJobOwnedDrift: + JOB_INDEXES: Final = ( + "-- CreateIndex\n" + 'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");\n' + "\n-- CreateIndex\n" + 'CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");\n' + ) + + def test_a_plain_spend_logs_table_only_loses_the_migration_job_indexes(self) -> None: + filtered: Final = ProxyExtrasDBManager._filter_migration_job_owned_drift( + _PARTITIONED_DRIFT_SQL + self.JOB_INDEXES, partitioned=False + ) + assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered + assert "LiteLLM_SpendLogs_api_key_startTime_idx" not in filtered + assert 'PRIMARY KEY ("request_id")' in filtered + + def test_a_partitioned_spend_logs_table_also_loses_its_partitioning_artifacts(self) -> None: + filtered: Final = ProxyExtrasDBManager._filter_migration_job_owned_drift( + _PARTITIONED_DRIFT_SQL + self.JOB_INDEXES, partitioned=True + ) + assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered + assert 'PRIMARY KEY ("request_id")' not in filtered + assert "LiteLLM_SpendLogs_legacy" not in filtered + assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered diff --git a/tests/litellm-proxy-extras/test_request_log_indexes.py b/tests/litellm-proxy-extras/test_request_log_indexes.py new file mode 100644 index 00000000000..f1d658285b6 --- /dev/null +++ b/tests/litellm-proxy-extras/test_request_log_indexes.py @@ -0,0 +1,149 @@ +import re +from pathlib import Path +from typing import Final + +import pytest +from litellm_proxy_extras.migration_recovery import is_inert_migration +from litellm_proxy_extras.request_log_indexes import ( + REQUEST_LOG_INDEXES, + RequestLogIndex, + filter_request_log_index_diff, +) + +PACKAGE: Final = Path(__file__).resolve().parents[2] / "litellm-proxy-extras" / "litellm_proxy_extras" +SCHEMA: Final = PACKAGE / "schema.prisma" +INERT_MIGRATIONS: Final = ( + "20260823000000_add_spend_logs_api_key_starttime_index", + "20260831120001_spend_logs_litellm_call_id_index", +) +CALL_ID_INDEX: Final = RequestLogIndex( + "LiteLLM_SpendLogs", "LiteLLM_SpendLogs_litellm_call_id_idx", '("litellm_call_id")' +) + + +def _prisma_indexes_of(schema: str, model: str) -> frozenset[str]: + """The index names Prisma derives for a model's @@index declarations: __idx.""" + body: Final = re.search(rf"model {model} \{{(.*?)\n\}}", schema, re.DOTALL) + assert body is not None, model + declarations: Final[tuple[str, ...]] = tuple( + match.group(1) for match in re.finditer(r"@@index\(\[([^\]]+)\]\)", body.group(1)) + ) + return frozenset( + f"{model}_{'_'.join(column.strip() for column in columns.split(','))}_idx" for columns in declarations + ) + + +class TestTheIndexList: + def test_every_migration_job_index_is_declared_in_the_prisma_schema_under_the_same_name(self) -> None: + schema: Final = SCHEMA.read_text() + for index in REQUEST_LOG_INDEXES: + assert index.name in _prisma_indexes_of(schema, index.table), index + + @pytest.mark.parametrize("name", INERT_MIGRATIONS) + def test_the_migrations_that_used_to_build_these_indexes_run_no_sql(self, name: str) -> None: + assert is_inert_migration((PACKAGE / "migrations" / name / "migration.sql").read_text()) + + +class TestIsInertMigration: + @pytest.mark.parametrize( + "script", + ( + "", + "-- only a comment\n", + "/* block */\n-- line\n", + "-- a semicolon; in a comment\n", + ";\n;", + "-- why\nSELECT 1;\n", + "select 1", + ), + ids=( + "empty", + "line-comment", + "both-comments", + "semicolon-in-comment", + "bare-separators", + "select-1", + "lowercase", + ), + ) + def test_comments_and_a_select_1_alone_are_inert(self, script: str) -> None: + assert is_inert_migration(script) is True + + @pytest.mark.parametrize( + "script", + ( + "SELECT 2;", + 'SELECT 1 FROM "LiteLLM_SpendLogs";', + '-- comment\nCREATE INDEX "ix" ON "t" ("a");', + "/* c */ ALTER TABLE t ADD COLUMN a TEXT", + 'SELECT 1; DROP INDEX "ix";', + ), + ids=("select-2", "select-from", "index-after-comment", "alter-after-block-comment", "drop-after-select-1"), + ) + def test_any_statement_is_not_inert(self, script: str) -> None: + assert is_inert_migration(script) is False + + +class TestPartitionIndexName: + def test_a_partition_gets_the_name_postgres_would_give_an_inherited_index(self) -> None: + assert CALL_ID_INDEX.partition_index_name("LiteLLM_SpendLogs_p2026_09") == ( + "LiteLLM_SpendLogs_p2026_09_litellm_call_id_idx" + ) + + def test_an_index_not_prefixed_by_its_table_keeps_its_whole_name(self) -> None: + index: Final = RequestLogIndex("LiteLLM_SpendLogs", "call_id_lookup", '("litellm_call_id")') + assert index.partition_index_name("LiteLLM_SpendLogs_pdefault") == "LiteLLM_SpendLogs_pdefault_call_id_lookup" + + def test_a_long_name_is_cut_to_63_bytes_with_a_digest_that_keeps_partitions_apart(self) -> None: + first: Final = CALL_ID_INDEX.partition_index_name("LiteLLM_SpendLogs_p" + "x" * 50 + "_2026_09") + second: Final = CALL_ID_INDEX.partition_index_name("LiteLLM_SpendLogs_p" + "x" * 50 + "_2026_10") + assert len(first.encode()) == 63 and len(second.encode()) == 63 + assert first != second + assert first.startswith("LiteLLM_SpendLogs_p") and first[-9] == "_" + + def test_the_byte_limit_counts_multibyte_characters(self) -> None: + name: Final = CALL_ID_INDEX.partition_index_name("é" * 40) + assert len(name.encode()) <= 63 and len(name) < 63 + + +class TestColumns: + def test_the_columns_are_the_quoted_names_of_the_definition_in_order(self) -> None: + index: Final = RequestLogIndex( + "LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", '("api_key", "startTime")' + ) + assert index.columns == ("api_key", "startTime") + + def test_every_migration_job_index_names_at_least_one_column(self) -> None: + assert all(index.columns for index in REQUEST_LOG_INDEXES) + + +DRIFT_WITH_BOTH_INDEXES: Final = ( + "-- CreateIndex\n" + 'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");\n' + "\n" + "-- CreateIndex\n" + 'CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");\n' +) + + +class TestFilterRequestLogIndexDiff: + def test_a_drift_script_that_only_creates_the_migration_job_indexes_becomes_empty(self) -> None: + assert filter_request_log_index_diff(DRIFT_WITH_BOTH_INDEXES) == "" + + def test_other_statements_survive_with_the_migration_job_indexes_removed(self) -> None: + other: Final = '-- AlterTable\nALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;\n' + filtered: Final = filter_request_log_index_diff(other + DRIFT_WITH_BOTH_INDEXES) + assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered + assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered + assert "LiteLLM_SpendLogs_api_key_startTime_idx" not in filtered + + def test_an_index_of_another_name_on_spend_logs_is_kept(self) -> None: + sql: Final = 'CREATE INDEX "LiteLLM_SpendLogs_end_user_idx" ON "LiteLLM_SpendLogs"("end_user");\n' + assert filter_request_log_index_diff(sql) == sql + + def test_a_drop_of_a_migration_job_index_is_kept_for_the_operator_to_see(self) -> None: + sql: Final = 'DROP INDEX "LiteLLM_SpendLogs_litellm_call_id_idx";\n' + assert filter_request_log_index_diff(sql) == sql + + def test_an_empty_script_stays_empty(self) -> None: + assert filter_request_log_index_diff("") == "" diff --git a/tests/proxy_migration_tests/test_db_schema_migration.py b/tests/proxy_migration_tests/test_db_schema_migration.py index 617ace37290..70498a3bcbd 100644 --- a/tests/proxy_migration_tests/test_db_schema_migration.py +++ b/tests/proxy_migration_tests/test_db_schema_migration.py @@ -5,7 +5,7 @@ import tempfile from pathlib import Path import pytest -from litellm_proxy_extras.utils import filter_hand_built_spend_logs_index_diff +from litellm_proxy_extras.request_log_indexes import filter_request_log_index_diff @pytest.mark.skipif( @@ -17,7 +17,9 @@ def test_schema_migration_in_sync(): Applies every committed migration to an empty database, then diffs the result against schema.prisma. A non-empty diff means the schema was changed without a - matching migration being generated. + matching migration being generated. The request-log indexes the migration job + builds are declared in the schema and deliberately absent from the migrations, + so those statements are filtered out before the diff is judged. """ db_url = os.environ["DATABASE_URL"] source_migrations_dir = Path( @@ -60,12 +62,15 @@ def test_schema_migration_in_sync(): text=True, ) - if diff.returncode == 2 and filter_hand_built_spend_logs_index_diff(diff.stdout).strip(): - pytest.fail( - "Schema changes detected that no migration captures. Run " - "`python litellm/ci_cd/run_migration.py `.\n\n" - + diff.stdout - ) - assert diff.returncode in (0, 2), f"prisma migrate diff errored: {diff.stderr}" + if diff.returncode == 2: + drift = filter_request_log_index_diff(diff.stdout) + if drift.strip(): + pytest.fail( + "Schema changes detected that no migration captures. Run " + "`python litellm/ci_cd/run_migration.py `.\n\n" + + drift + ) + else: + assert diff.returncode == 0, f"prisma migrate diff errored: {diff.stderr}" finally: shutil.rmtree(temp_base, ignore_errors=True) diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py index 741fa7386df..3f68b8d025a 100644 --- a/tests/proxy_migration_tests/test_invalid_index_repair.py +++ b/tests/proxy_migration_tests/test_invalid_index_repair.py @@ -1,12 +1,15 @@ import os import threading +import time import uuid from collections.abc import Iterator, Mapping from types import MappingProxyType from typing import Final import pytest -from litellm_proxy_extras.utils import INDEX_REPAIR_ADVISORY_LOCK_KEY, ProxyExtrasDBManager +from litellm_proxy_extras.migration_lock import MIGRATION_LOCK_KEY +from litellm_proxy_extras.request_log_indexes import REQUEST_LOG_INDEXES +from litellm_proxy_extras.utils import INDEX_REPAIR_ADVISORY_LOCK_KEY, ProxyExtrasDBManager, _InvalidIndex psycopg = pytest.importorskip("psycopg") @@ -20,6 +23,7 @@ requires_db: Final = pytest.mark.skipif( HEALTH_TABLE: Final = "LiteLLM_HealthCheckTable" HEALTH_INDEX: Final = "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx" HEALTH_INDEX_COLUMNS: Final = '"model_id", "model_name", "checked_at" DESC' +SECOND_HEALTH_INDEX: Final = "LiteLLM_HealthCheckTable_model_name_idx" LOOKALIKE_TABLE: Final = "LiteLLMLookalikeTable" LOOKALIKE_INDEX: Final = "LiteLLMLookalikeTable_id_idx" PARTITIONED_TABLE: Final = "LiteLLM_PartitionedTable" @@ -112,6 +116,48 @@ def test_repair_rebuilds_invalid_litellm_indexes_and_leaves_lookalike_tables_alo assert _index_validity(scratch_schema) == {HEALTH_INDEX: True, LOOKALIKE_INDEX: False} +@requires_db +@pytest.mark.parametrize("partitioned", (False, True)) +def test_startup_repair_defers_managed_indexes_but_repairs_other_indexes( + scratch_schema: str, partitioned: bool +) -> None: + table: Final = "LiteLLM_DifferentlyNamedLeaf" if partitioned else "LiteLLM_SpendLogs" + managed: Final = REQUEST_LOG_INDEXES[0] + name: Final = managed.partition_index_name(table) if partitioned else managed.name + with psycopg.connect(_base_url(), autocommit=True) as connection: + connection.execute( + f'CREATE TABLE "{scratch_schema}"."LiteLLM_SpendLogs" ' + '(api_key TEXT, "startTime" TIMESTAMPTZ, litellm_call_id TEXT)' + + (' PARTITION BY RANGE ("startTime")' if partitioned else '') + ) + if partitioned: + connection.execute( + f'CREATE TABLE "{scratch_schema}"."{table}" PARTITION OF ' + f'"{scratch_schema}"."LiteLLM_SpendLogs" FOR VALUES FROM (MINVALUE) TO (MAXVALUE)' + ) + _leave_invalid_index(scratch_schema, table, name, managed.definition.strip("()")) + _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS) + + assert ProxyExtrasDBManager.repair_invalid_indexes(skip_request_log_indexes=True) is True + assert _index_validity(scratch_schema) == {name: False, HEALTH_INDEX: True} + + assert ProxyExtrasDBManager.build_request_log_indexes() is True + assert all(_index_validity(scratch_schema).values()) + + +@requires_db +def test_database_setup_leaves_managed_repair_to_the_request_log_builder(fresh_database: str) -> None: + assert ProxyExtrasDBManager.setup_database(use_migrate=True) is True + managed: Final = REQUEST_LOG_INDEXES[0] + _leave_invalid_index(fresh_database, managed.table, managed.name, managed.definition.strip("()")) + + assert ProxyExtrasDBManager.setup_database(use_migrate=True) is True + assert _index_validity(fresh_database)[managed.name] is False + + assert ProxyExtrasDBManager.build_request_log_indexes() is True + assert all(_index_validity(fresh_database)[index.name] for index in REQUEST_LOG_INDEXES) + + @requires_db def test_repair_drops_leftovers_of_interrupted_rebuilds(scratch_schema: str) -> None: _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS) @@ -167,7 +213,74 @@ def test_repair_yields_to_the_replica_holding_the_repair_lock(scratch_schema: st @requires_db -def test_repair_gives_up_on_a_blocked_rebuild_and_finishes_it_on_the_next_startup(scratch_schema: str) -> None: +def test_repair_yields_to_the_migration_job_building_indexes_under_the_migration_lock(scratch_schema: str) -> None: + """A migration job's index build holds the migration lock while its CREATE INDEX CONCURRENTLY + is cataloged as invalid; the repair must not rebuild that in-flight index.""" + _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS) + + with psycopg.connect(_base_url(), autocommit=True) as index_builder: + index_builder.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,)) + assert ProxyExtrasDBManager.repair_invalid_indexes() is False + assert _index_validity(scratch_schema) == {HEALTH_INDEX: False} + + assert ProxyExtrasDBManager.repair_invalid_indexes() is True + assert _index_validity(scratch_schema) == {HEALTH_INDEX: True} + + +def _hold_migration_lock_once_free(release: threading.Event) -> None: + with psycopg.connect(_base_url(), autocommit=True) as resolver: + resolver.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,)) + release.wait(timeout=60) + + +def _wait_until_a_session_queues_for_the_migration_lock() -> None: + with psycopg.connect(_base_url(), autocommit=True) as conn: + for _ in range(200): + queued: Final = conn.execute( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND NOT granted " + "AND classid = %s AND objid = %s", + (MIGRATION_LOCK_KEY >> 32, MIGRATION_LOCK_KEY & 0xFFFFFFFF), + ).fetchone() + if queued is not None and queued[0]: + return + time.sleep(0.05) + pytest.fail("no session queued for the migration lock") + + +@requires_db +def test_repair_releases_the_migration_lock_between_indexes_so_a_booting_resolver_gets_in( + scratch_schema: str, +) -> None: + """A v2 resolver on another replica waits for the migration lock; with two invalid + indexes to rebuild it must get the lock after the first REINDEX, not after both.""" + _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS) + _leave_invalid_index(scratch_schema, HEALTH_TABLE, SECOND_HEALTH_INDEX, '"model_name"') + release: Final = threading.Event() + resolver: Final = threading.Thread(target=_hold_migration_lock_once_free, args=(release,)) + + def repair_then_let_a_resolver_queue_for_the_lock( + conn: "psycopg.Connection[tuple[str, str, str]]", index: _InvalidIndex + ) -> None: + ProxyExtrasDBManager._repair_index(conn, index) + if not resolver.is_alive(): + resolver.start() + _wait_until_a_session_queues_for_the_migration_lock() + + try: + assert ( + ProxyExtrasDBManager.repair_invalid_indexes(repair=repair_then_let_a_resolver_queue_for_the_lock) is False + ) + assert sorted(_index_validity(scratch_schema).values()) == [False, True] + finally: + release.set() + resolver.join() + + assert ProxyExtrasDBManager.repair_invalid_indexes() is True + assert _index_validity(scratch_schema) == {HEALTH_INDEX: True, SECOND_HEALTH_INDEX: True} + + +@requires_db +def test_repair_gives_up_on_a_blocked_rebuild_and_finishes_it_on_the_next_boot(scratch_schema: str) -> None: _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS) with psycopg.connect(_base_url()) as pin: diff --git a/tests/proxy_migration_tests/test_request_log_indexes.py b/tests/proxy_migration_tests/test_request_log_indexes.py new file mode 100644 index 00000000000..015492c7040 --- /dev/null +++ b/tests/proxy_migration_tests/test_request_log_indexes.py @@ -0,0 +1,966 @@ +import os +import queue +import shutil +import subprocess +import sys +import threading +import time +import uuid +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import psycopg +import pytest +from litellm_proxy_extras import request_log_indexes +from litellm_proxy_extras.migration_lock import MIGRATION_LOCK_KEY, migration_lock +from litellm_proxy_extras.migration_recovery import roll_back_failed_inert_migration +from litellm_proxy_extras.request_log_indexes import ( + REQUEST_LOG_INDEXES, + RequestLogIndex, + build_index_on_partitioned_table, + ensure_request_log_indexes, +) +from litellm_proxy_extras.utils import ProxyExtrasDBManager +from psycopg import sql +from psycopg.abc import Params, QueryNoTemplate +from psycopg.rows import class_row + +pytestmark = pytest.mark.timeout(900) + +requires_db: Final = pytest.mark.skipif( + "DATABASE_URL" not in os.environ, + reason="requires a postgres database (DATABASE_URL)", +) + +REPO: Final = Path(__file__).resolve().parents[2] +PACKAGE: Final = REPO / "litellm-proxy-extras" / "litellm_proxy_extras" +PARTITION_SCRIPT: Final = REPO / "db_scripts" / "partition_spend_logs.sql" +API_KEY_INDEX_MIGRATION: Final = "20260823000000_add_spend_logs_api_key_starttime_index" +CALL_ID_INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index" +API_KEY_INDEX: Final = "LiteLLM_SpendLogs_api_key_startTime_idx" +CALL_ID_INDEX: Final = "LiteLLM_SpendLogs_litellm_call_id_idx" +PARTITIONED_PARENT_ERROR: Final = 'cannot create index on partitioned table "LiteLLM_SpendLogs" concurrently' +ORIGINAL_MIGRATION_SQL: Final = MappingProxyType( + { + API_KEY_INDEX_MIGRATION: ( + "-- CreateIndex\n" + 'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" ' + 'ON "LiteLLM_SpendLogs"("api_key", "startTime");\n' + ), + CALL_ID_INDEX_MIGRATION: ( + "-- CreateIndex\n" + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" ' + 'ON "LiteLLM_SpendLogs"("litellm_call_id");\n' + ), + } +) +CALL_ID_INDEX_DEFINITION: Final = next(index for index in REQUEST_LOG_INDEXES if index.name == CALL_ID_INDEX) +RELEASES: Final = pytest.mark.parametrize( + "release", (API_KEY_INDEX_MIGRATION, CALL_ID_INDEX_MIGRATION), ids=("v1.102.1", "v1.103.0") +) +PARTITIONS: Final = MappingProxyType( + { + "LiteLLM_SpendLogs_p2026_08": ("2026-08-01", "2026-09-01"), + "LiteLLM_SpendLogs_p2026_09": ("2026-09-01", "2026-10-01"), + } +) +DEFAULT_PARTITION: Final = "LiteLLM_SpendLogs_pdefault" +ROWS_PER_PARTITION: Final = 200 +RESOLVERS: Final = pytest.mark.parametrize("use_v2_resolver", (True, False), ids=("v2", "v1")) + + +def _base_url() -> str: + return os.environ["DATABASE_URL"].split("?")[0] + + +def _migrate_deploy(database_url: str, schema: Path) -> "subprocess.CompletedProcess[str]": + return subprocess.run( + [sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(schema)], + capture_output=True, + text=True, + env={**os.environ, "DATABASE_URL": database_url}, + ) + + +def _release_layout(prisma_dir: Path, before: str) -> Path: + """The shipped migrations older than `before`, with the two index migrations written + the way the releases that shipped them did: the Prisma layout of a proxy on that release.""" + (prisma_dir / "migrations").mkdir(parents=True) + shutil.copy(PACKAGE / "schema.prisma", prisma_dir / "schema.prisma") + for migration in sorted((PACKAGE / "migrations").iterdir()): + if migration.is_dir() and migration.name < before: + shutil.copytree(migration, prisma_dir / "migrations" / migration.name) + for name, original in ORIGINAL_MIGRATION_SQL.items(): + if (prisma_dir / "migrations" / name).is_dir(): + (prisma_dir / "migrations" / name / "migration.sql").write_text(original) + return prisma_dir / "schema.prisma" + + +def _deploy_release(database_url: str, prisma_dir: Path, before: str) -> None: + deployed: Final = _migrate_deploy(database_url, _release_layout(prisma_dir, before)) + assert deployed.returncode == 0, deployed.stderr + + +def _insert_spend_log( + conn: "psycopg.Connection[tuple[object, ...]]", request_id: str, day: str, table: str = "LiteLLM_SpendLogs" +) -> None: + conn.execute( + sql.SQL( + 'INSERT INTO {} ("request_id", "call_type", "startTime", "endTime", "api_key") VALUES (%s, %s, %s, %s, %s)' + ).format(sql.Identifier(table)), + (request_id, "acompletion", day, day, f"key-{request_id[-1]}"), + ) + + +def _partition_spend_logs(database_url: str) -> None: + with psycopg.connect(database_url, autocommit=True) as conn: + conn.execute(PARTITION_SCRIPT.read_bytes()) + for partition, (start, stop) in PARTITIONS.items(): + conn.execute( + sql.SQL('CREATE TABLE {} PARTITION OF "LiteLLM_SpendLogs" FOR VALUES FROM ({}) TO ({})').format( + sql.Identifier(partition), sql.Literal(start), sql.Literal(stop) + ) + ) + for row in range(ROWS_PER_PARTITION): + _insert_spend_log(conn, f"{partition}-{row}", start) + for row in range(ROWS_PER_PARTITION): + _insert_spend_log(conn, f"default-{row}", "2020-01-01") + + +@pytest.fixture +def release() -> str: + """The first migration a database has not applied yet; the v1.103.0 shape unless a test parametrizes it.""" + return CALL_ID_INDEX_MIGRATION + + +@pytest.fixture +def scratch_database(release: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[str]: + """A deployment stopped before `release`, with DATABASE_URL pointed at it so + ProxyExtrasDBManager upgrades it like a booting proxy.""" + admin_url: Final = _base_url() + name: Final = f"spend_logs_index_{uuid.uuid4().hex[:8]}" + with psycopg.connect(admin_url, autocommit=True) as conn: + conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name))) + database_url: Final = f"{admin_url.rsplit('/', 1)[0]}/{name}" + try: + _deploy_release(database_url, tmp_path / "prisma", release) + monkeypatch.delenv("DIRECT_URL", raising=False) + monkeypatch.setenv("DATABASE_URL", database_url) + yield database_url + finally: + with psycopg.connect(admin_url, autocommit=True) as conn: + conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(name))) + + +@pytest.fixture +def partitioned_database(scratch_database: str) -> str: + _partition_spend_logs(scratch_database) + return scratch_database + + +def _fail_the_call_id_migration_like_the_shipped_release(database_url: str, tmp_path: Path) -> None: + """Boot the original v1.103.0 layout once: its CONCURRENTLY statement fails on the + partitioned parent and leaves the call_id ledger row unfinished.""" + failed: Final = _migrate_deploy(database_url, _release_layout(tmp_path / "v1.103.0", "99999999999999")) + assert failed.returncode != 0 and PARTITIONED_PARENT_ERROR in failed.stderr, failed.stderr + assert _ledger(database_url)[CALL_ID_INDEX_MIGRATION] == (False, False) + + +@dataclass(frozen=True, slots=True) +class _IndexRow: + name: str + valid: bool + + +@dataclass(frozen=True, slots=True) +class _AttachedRow: + table: str + index: str + + +@dataclass(frozen=True, slots=True) +class _LedgerRow: + name: str + finished: bool + rolled_back: bool + + +@dataclass(frozen=True, slots=True) +class _OidRow: + name: str + oid: int + + +def _index_validity(database_url: str, suffix: str) -> Mapping[str, bool]: + """index name -> indisvalid for every index ending in `suffix` on the SpendLogs parent or one of its partitions.""" + with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_IndexRow)) as cursor: + rows: Final = cursor.execute( + "SELECT c.relname AS name, i.indisvalid AS valid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid " + "WHERE c.relname LIKE %s AND (i.indrelid = to_regclass('\"LiteLLM_SpendLogs\"') OR i.indrelid IN " + "(SELECT inhrelid FROM pg_inherits WHERE inhparent = to_regclass('\"LiteLLM_SpendLogs\"'))) " + "ORDER BY c.relname", + (f"%{suffix}",), + ).fetchall() + return MappingProxyType({row.name: row.valid for row in rows}) + + +def _attached_children(database_url: str, parent_index: str) -> frozenset[tuple[str, str]]: + """(partition, child index) pairs attached under the parent index.""" + with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_AttachedRow)) as cursor: + rows: Final = cursor.execute( + 'SELECT t.relname AS "table", c.relname AS index FROM pg_inherits i ' + "JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_index x ON x.indexrelid = c.oid " + "JOIN pg_class t ON t.oid = x.indrelid " + "WHERE i.inhparent = to_regclass(%s)", + (f'"{parent_index}"',), + ).fetchall() + return frozenset((row.table, row.index) for row in rows) + + +@dataclass(frozen=True, slots=True) +class _TableRow: + name: str + + +def _indexed_table(database_url: str, index: str) -> "str | None": + with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_TableRow)) as cursor: + row: Final = cursor.execute( + "SELECT t.relname AS name FROM pg_index x JOIN pg_class t ON t.oid = x.indrelid " + "WHERE x.indexrelid = to_regclass(%s)", + (f'"{index}"',), + ).fetchone() + return None if row is None else row.name + + +def _ledger(database_url: str) -> Mapping[str, tuple[bool, bool]]: + """migration name -> (finished, rolled back) for the newest ledger row of each migration.""" + with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_LedgerRow)) as cursor: + rows: Final = cursor.execute( + "SELECT DISTINCT ON (migration_name) migration_name AS name, finished_at IS NOT NULL AS finished, " + "rolled_back_at IS NOT NULL AS rolled_back FROM _prisma_migrations ORDER BY migration_name, started_at DESC" + ).fetchall() + return MappingProxyType({row.name: (row.finished, row.rolled_back) for row in rows}) + + +def _index_oids(database_url: str) -> Mapping[str, int]: + """index name -> oid for every index on the SpendLogs parent or one of its partitions; a rebuild changes the oid.""" + with psycopg.connect(database_url) as conn, conn.cursor(row_factory=class_row(_OidRow)) as cursor: + rows: Final = cursor.execute( + "SELECT c.relname AS name, c.oid::int AS oid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid " + "WHERE i.indrelid = to_regclass('\"LiteLLM_SpendLogs\"') OR i.indrelid IN " + "(SELECT inhrelid FROM pg_inherits WHERE inhparent = to_regclass('\"LiteLLM_SpendLogs\"'))" + ).fetchall() + return MappingProxyType({row.name: row.oid for row in rows}) + + +def _migration_job(use_v2_resolver: bool) -> bool: + return ProxyExtrasDBManager.run_migration_job(use_migrate=True, use_v2_resolver=use_v2_resolver) + + +def _assert_no_pending_migrations(database_url: str) -> None: + status: Final = _migrate_deploy(database_url, PACKAGE / "schema.prisma") + assert status.returncode == 0 and "No pending migrations" in status.stdout, status.stdout + status.stderr + + +def _assert_every_ledger_row_is_finished(database_url: str) -> Mapping[str, tuple[bool, bool]]: + ledger: Final = _ledger(database_url) + assert ledger[API_KEY_INDEX_MIGRATION] == (True, False) and ledger[CALL_ID_INDEX_MIGRATION] == (True, False) + assert all(finished and not rolled_back for finished, rolled_back in ledger.values()), ledger + return ledger + + +def _expected_children(partitions: tuple[str, ...], suffix: str) -> frozenset[tuple[str, str]]: + return frozenset((partition, f"{partition}_{suffix}") for partition in partitions) + + +def _assert_index_covers_every_partition(database_url: str, parent_index: str, suffix: str) -> None: + partitions: Final = (*PARTITIONS, DEFAULT_PARTITION) + assert _index_validity(database_url, suffix) == {parent_index: True} | {f"{p}_{suffix}": True for p in partitions} + assert _attached_children(database_url, parent_index) == _expected_children(partitions, suffix) + + +@requires_db +@RESOLVERS +@RELEASES +def test_a_partitioned_spend_logs_upgrade_builds_both_indexes_per_partition_and_a_rerun_is_idempotent( + partitioned_database: str, use_v2_resolver: bool +) -> None: + assert _migration_job(use_v2_resolver) is True + + _assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx") + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + ledger: Final = _assert_every_ledger_row_is_finished(partitioned_database) + _assert_no_pending_migrations(partitioned_database) + oids: Final = _index_oids(partitioned_database) + + with psycopg.connect(partitioned_database, autocommit=True) as conn: + conn.execute( + 'CREATE TABLE "LiteLLM_SpendLogs_p2026_10" PARTITION OF "LiteLLM_SpendLogs" ' + "FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')" + ) + inherited: Final = frozenset( + ("LiteLLM_SpendLogs_p2026_10", f"LiteLLM_SpendLogs_p2026_10_{suffix}") + for suffix in ("api_key_startTime_idx", "litellm_call_id_idx") + ) + attached: Final = _attached_children(partitioned_database, API_KEY_INDEX) | _attached_children( + partitioned_database, CALL_ID_INDEX + ) + assert inherited <= attached, attached + + assert _migration_job(use_v2_resolver) is True + assert _ledger(partitioned_database) == ledger + assert {name: oid for name, oid in _index_oids(partitioned_database).items() if name in oids} == oids + + +@requires_db +@RESOLVERS +@RELEASES +def test_a_plain_spend_logs_upgrade_builds_both_indexes_and_a_second_job_run_rebuilds_nothing( + scratch_database: str, use_v2_resolver: bool +) -> None: + with psycopg.connect(scratch_database, autocommit=True) as conn: + for row in range(ROWS_PER_PARTITION): + _insert_spend_log(conn, f"flat-{row}", "2026-09-01") + + assert _migration_job(use_v2_resolver) is True + + assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True} + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + _assert_every_ledger_row_is_finished(scratch_database) + _assert_no_pending_migrations(scratch_database) + oids: Final = _index_oids(scratch_database) + + assert _migration_job(use_v2_resolver) is True + assert _index_oids(scratch_database) == oids + + +@requires_db +@RESOLVERS +def test_a_database_that_applied_the_original_migration_files_sees_no_pending_migrations_and_no_rebuild( + scratch_database: str, use_v2_resolver: bool, tmp_path: Path +) -> None: + """A plain table upgraded on v1.103.0 applied both original files. The inert files in + this build must neither re-run nor fail those rows, and the migration job must keep the + indexes the migrations built.""" + deployed: Final = _migrate_deploy(scratch_database, _release_layout(tmp_path / "v1.103.0", "99999999999999")) + assert deployed.returncode == 0, deployed.stderr + before: Final = _ledger(scratch_database) + assert before[API_KEY_INDEX_MIGRATION] == (True, False) and before[CALL_ID_INDEX_MIGRATION] == (True, False) + oids: Final = _index_oids(scratch_database) + assert {API_KEY_INDEX, CALL_ID_INDEX} <= set(oids) + + _assert_no_pending_migrations(scratch_database) + assert _migration_job(use_v2_resolver) is True + + assert _ledger(scratch_database) == before + assert _index_oids(scratch_database) == oids + + +@requires_db +@RESOLVERS +def test_a_failed_call_id_ledger_row_from_a_v1_103_boot_is_rolled_back_and_the_inert_file_applied( + partitioned_database: str, use_v2_resolver: bool, tmp_path: Path +) -> None: + _fail_the_call_id_migration_like_the_shipped_release(partitioned_database, tmp_path) + + assert _migration_job(use_v2_resolver) is True + + _assert_every_ledger_row_is_finished(partitioned_database) + _assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx") + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + _assert_no_pending_migrations(partitioned_database) + with psycopg.connect(partitioned_database) as conn: + rows: Final = conn.execute( + "SELECT finished_at IS NOT NULL, rolled_back_at IS NOT NULL FROM _prisma_migrations " + "WHERE migration_name = %s ORDER BY started_at", + (CALL_ID_INDEX_MIGRATION,), + ).fetchall() + assert rows == [(False, True), (True, False)], rows + + +@requires_db +def test_a_failed_row_whose_migration_still_runs_sql_in_this_build_is_left_for_the_operator( + partitioned_database: str, tmp_path: Path +) -> None: + _fail_the_call_id_migration_like_the_shipped_release(partitioned_database, tmp_path) + still_building: Final = tmp_path / "edited" / CALL_ID_INDEX_MIGRATION / "migration.sql" + still_building.parent.mkdir(parents=True) + still_building.write_text(ORIGINAL_MIGRATION_SQL[CALL_ID_INDEX_MIGRATION]) + + with migration_lock(partitioned_database) as coordinator: + assert roll_back_failed_inert_migration(coordinator, "public", still_building) is False + + assert _ledger(partitioned_database)[CALL_ID_INDEX_MIGRATION] == (False, False) + + +@requires_db +def test_a_migration_without_a_failed_row_is_not_touched(partitioned_database: str) -> None: + inert: Final = PACKAGE / "migrations" / CALL_ID_INDEX_MIGRATION / "migration.sql" + before: Final = _ledger(partitioned_database) + + with migration_lock(partitioned_database) as coordinator: + assert roll_back_failed_inert_migration(coordinator, "public", inert) is False + + assert _ledger(partitioned_database) == before + + +def _pin_a_snapshot_on(database_url: str, table: str) -> "psycopg.Connection[tuple[object, ...]]": + pin: Final = psycopg.connect(database_url) + pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ + pin.execute(sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table))) + return pin + + +def _leave_an_invalid_index(database_url: str, name: str, table: str, column: str) -> None: + with _pin_a_snapshot_on(database_url, table): + with psycopg.connect(database_url, autocommit=True) as builder: + builder.execute("SET statement_timeout = '1s'") + with pytest.raises(psycopg.errors.QueryCanceled): + builder.execute( + sql.SQL("CREATE INDEX CONCURRENTLY {} ON {} ({})").format( + sql.Identifier(name), sql.Identifier(table), sql.Identifier(column) + ) + ) + + +@requires_db +def test_an_invalid_index_of_the_managed_name_on_a_plain_table_is_rebuilt(scratch_database: str) -> None: + _leave_an_invalid_index(scratch_database, CALL_ID_INDEX, "LiteLLM_SpendLogs", "litellm_call_id") + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: False} + + assert ensure_request_log_indexes(scratch_database, "public") is True + + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + + +def _rebuild_as_another_replica(database_url: str, name: str, table: str, column: str) -> int: + """Drop and rebuild the index from a second connection, as a replica that won the + race would, and return the oid of the index it built.""" + with psycopg.connect(database_url, autocommit=True) as other_replica: + other_replica.execute(sql.SQL("DROP INDEX {}").format(sql.Identifier(name))) + other_replica.execute( + sql.SQL("CREATE INDEX {} ON {} ({})").format( + sql.Identifier(name), sql.Identifier(table), sql.Identifier(column) + ) + ) + return _index_oids(database_url)[name] + + +def _connecting_with_another_replica_acting_first( + statement: str, other_replica: Callable[[QueryNoTemplate], None] +) -> Callable[[str], "psycopg.Connection[tuple[object, ...]]"]: + """A connect function whose cursors let `other_replica` act, once, right before the + first statement containing `statement` runs: the interleaving two replicas booting + together can produce, made deterministic.""" + raced: Final = threading.Event() + + class _RacedCursor(psycopg.Cursor[tuple[object, ...]]): + def execute( # pyright: ignore[reportIncompatibleMethodOverride] # the builder never runs a Template query + self, + query: QueryNoTemplate, + params: "Params | None" = None, + *, + prepare: "bool | None" = None, + binary: "bool | None" = None, + ) -> "_RacedCursor": + text: Final = query.as_string(self.connection) if isinstance(query, sql.Composable) else query + if isinstance(text, str) and statement in text and not raced.is_set(): + raced.set() + other_replica(query) + return super().execute(query, params, prepare=prepare, binary=binary) + + def connect(database_url: str) -> "psycopg.Connection[tuple[object, ...]]": + return psycopg.connect(database_url, autocommit=True, cursor_factory=_RacedCursor) + + return connect + + +@requires_db +def test_an_index_another_replica_made_valid_before_the_lock_was_taken_is_kept(scratch_database: str) -> None: + """Two replicas boot against the same invalid index. The one that takes the lock + second must read the catalog again under it, or it drops the valid index the first + one just finished and starts the whole build over.""" + _leave_an_invalid_index(scratch_database, CALL_ID_INDEX, "LiteLLM_SpendLogs", "litellm_call_id") + theirs: Final[queue.SimpleQueue[int]] = queue.SimpleQueue() + connect: Final = _connecting_with_another_replica_acting_first( + "pg_try_advisory_lock", + lambda _: theirs.put( + _rebuild_as_another_replica(scratch_database, CALL_ID_INDEX, "LiteLLM_SpendLogs", "litellm_call_id") + ), + ) + + assert ensure_request_log_indexes(scratch_database, "public", (CALL_ID_INDEX_DEFINITION,), connect) is True + + assert _index_oids(scratch_database)[CALL_ID_INDEX] == theirs.get_nowait() + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + + +@requires_db +def test_a_child_index_another_replica_attached_first_is_not_attached_twice(partitioned_database: str) -> None: + """A replica that reaches the attach step after another one attached the same child + relies on ATTACH PARTITION being a no-op for an index already under that parent + (PostgreSQL 14 ALTER INDEX, ATExecAttachPartitionIdx, checked 2026-10-01); this test + is where that would surface if a future version or a code change made it an error.""" + + def attach_as_another_replica(statement: QueryNoTemplate) -> None: + with psycopg.connect(partitioned_database, autocommit=True) as other_replica: + other_replica.execute(statement) + + connect: Final = _connecting_with_another_replica_acting_first("ATTACH PARTITION", attach_as_another_replica) + + assert ensure_request_log_indexes(partitioned_database, "public", (CALL_ID_INDEX_DEFINITION,), connect) is True + + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_an_invalid_child_index_left_by_an_interrupted_build_is_rebuilt_and_attached( + partitioned_database: str, +) -> None: + partition: Final = "LiteLLM_SpendLogs_p2026_08" + child: Final = f"{partition}_litellm_call_id_idx" + _leave_an_invalid_index(partitioned_database, child, partition, "litellm_call_id") + assert _index_validity(partitioned_database, "litellm_call_id_idx") == {child: False} + + assert ensure_request_log_indexes(partitioned_database, "public") is True + + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_partitioning_a_table_the_index_job_already_covered_moves_its_call_id_index_aside(scratch_database: str) -> None: + """The runbook renames the Prisma indexes off the legacy table so the partitioned table can take their + names. The call-id index is built by the index job, not Prisma, and a deployment that partitions after + that job ran kept the name on the legacy table, so every later job run reported it as on another table.""" + assert ensure_request_log_indexes(scratch_database, "public") is True + assert _indexed_table(scratch_database, CALL_ID_INDEX) == "LiteLLM_SpendLogs" + + _partition_spend_logs(scratch_database) + + assert _indexed_table(scratch_database, "LiteLLM_SpendLogs_legacy_litellm_call_id_idx") == "LiteLLM_SpendLogs_legacy" + assert ensure_request_log_indexes(scratch_database, "public") is True + _assert_index_covers_every_partition(scratch_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_an_index_of_that_name_on_another_table_is_left_alone_and_reported(partitioned_database: str) -> None: + with psycopg.connect(partitioned_database, autocommit=True) as conn: + conn.execute(f'CREATE INDEX "{CALL_ID_INDEX}" ON "LiteLLM_ErrorLogs" ("request_id")') + + assert ensure_request_log_indexes(partitioned_database, "public") is False + + assert _index_validity(partitioned_database, "litellm_call_id_idx") == {} + assert _indexed_table(partitioned_database, CALL_ID_INDEX) == "LiteLLM_ErrorLogs" + _assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx") + + +@requires_db +def test_an_invalid_index_of_a_child_name_on_another_table_is_not_dropped(partitioned_database: str) -> None: + child: Final = "LiteLLM_SpendLogs_p2026_08_litellm_call_id_idx" + _leave_an_invalid_index(partitioned_database, child, "LiteLLM_ErrorLogs", "request_id") + + assert ensure_request_log_indexes(partitioned_database, "public") is False + + assert _indexed_table(partitioned_database, child) == "LiteLLM_ErrorLogs" + assert _attached_children(partitioned_database, CALL_ID_INDEX) == frozenset() + + +@requires_db +def test_a_process_holding_the_migration_lock_makes_the_build_wait_for_the_next_job_run(scratch_database: str) -> None: + with psycopg.connect(scratch_database, autocommit=True) as other_replica: + other_replica.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,)) + assert ensure_request_log_indexes(scratch_database, "public") is False + assert _index_validity(scratch_database, "litellm_call_id_idx") == {} + + assert ensure_request_log_indexes(scratch_database, "public") is True + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + + +@requires_db +@RESOLVERS +def test_a_migration_job_that_could_not_build_the_indexes_reports_failure_and_succeeds_when_rerun( + scratch_database: str, use_v2_resolver: bool +) -> None: + """The migration job waits for the build and exits by run_migration_job's result; a job + that exits 0 with the indexes missing would leave the table unindexed until the next + deploy or until a serving proxy's background build gets to them.""" + with psycopg.connect(scratch_database, autocommit=True) as other_replica: + other_replica.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_KEY,)) + assert _migration_job(use_v2_resolver) is False + _assert_every_ledger_row_is_finished(scratch_database) + assert _index_validity(scratch_database, "litellm_call_id_idx") == {} + + assert _migration_job(use_v2_resolver) is True + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True} + + +@requires_db +@RESOLVERS +def test_the_serving_proxy_setup_applies_the_inert_migrations_and_builds_no_index( + partitioned_database: str, use_v2_resolver: bool +) -> None: + """setup_database alone applies the inert files and builds nothing, so a serving proxy's + readiness is never held up by an index build; the build it starts afterwards, or the + migration job, is what puts the indexes in place.""" + api_key_index_before: Final = _index_validity(partitioned_database, "api_key_startTime_idx") + assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True + + _assert_every_ledger_row_is_finished(partitioned_database) + _assert_no_pending_migrations(partitioned_database) + assert _index_validity(partitioned_database, "litellm_call_id_idx") == {} + assert _index_validity(partitioned_database, "api_key_startTime_idx") == api_key_index_before + + assert _migration_job(use_v2_resolver) is True + _assert_index_covers_every_partition(partitioned_database, API_KEY_INDEX, "api_key_startTime_idx") + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_a_role_that_may_not_create_indexes_is_logged_and_left_for_the_next_job_run( + scratch_database: str, caplog: pytest.LogCaptureFixture +) -> None: + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC") + conn.execute("CREATE ROLE spend_logs_reader LOGIN PASSWORD 'reader'") + conn.execute("GRANT USAGE ON SCHEMA public TO spend_logs_reader") + conn.execute('GRANT SELECT ON "LiteLLM_SpendLogs" TO spend_logs_reader') + reader_url: Final = scratch_database.replace("postgres:postgres@", "spend_logs_reader:reader@", 1) + try: + with caplog.at_level("WARNING", logger="litellm_proxy_extras"): + assert ensure_request_log_indexes(reader_url, "public") is False + finally: + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute("DROP OWNED BY spend_logs_reader") + conn.execute("DROP ROLE spend_logs_reader") + assert "leaving them for the next index build" in caplog.text + assert _index_validity(scratch_database, "litellm_call_id_idx") == {} + + +@requires_db +def test_inserts_keep_flowing_while_the_partition_indexes_build(partitioned_database: str) -> None: + """With a write open on one partition, the parent index goes on ONLY the parent and + the CONCURRENTLY child build waits for that write without blocking new INSERTs. A + plain CREATE INDEX on the parent would wait for the same write while holding SHARE + on the parent, queueing every new INSERT behind it.""" + outcome: Final[list[bool]] = [] # mutable-ok: the builder thread hands its result back through it + with psycopg.connect(partitioned_database) as writer: + _insert_spend_log(writer, "LiteLLM_SpendLogs_p2026_08-open", "2026-08-15", table="LiteLLM_SpendLogs_p2026_08") + builder_thread: Final = threading.Thread( + target=lambda: outcome.append(_build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION)) + ) + builder_thread.start() + try: + _wait_until_the_build_is_waiting(partitioned_database) + with psycopg.connect(partitioned_database, autocommit=True) as late_writer: + late_writer.execute("SET lock_timeout = '1s'") + _insert_spend_log(late_writer, "LiteLLM_SpendLogs_p2026_08-late", "2026-08-16") + finally: + writer.commit() + builder_thread.join() + assert outcome == [True] + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +def _insert_for(database_url: str, seconds: float) -> None: + with psycopg.connect(database_url, autocommit=True) as conn: + conn.execute("SET lock_timeout = '1s'") + deadline: Final = time.monotonic() + seconds + while time.monotonic() < deadline: + _insert_spend_log(conn, f"lock-test-{uuid.uuid4().hex}", "2026-08-16") + time.sleep(0.05) + + +def _wait_for_blocked_ddl(database_url: str, query_pattern: str) -> bool: + with psycopg.connect(database_url, autocommit=True) as conn: + deadline: Final = time.monotonic() + 10 + while time.monotonic() < deadline: + if conn.execute( + "SELECT 1 FROM pg_stat_activity WHERE wait_event_type = 'Lock' AND query ILIKE %s", + (query_pattern,), + ).fetchone(): + return True + time.sleep(0.01) + return False + + +@requires_db +def test_inserts_are_never_held_back_while_the_parent_index_waits_for_an_open_write( + partitioned_database: str, +) -> None: + outcome: Final[list[bool]] = [] # mutable-ok: the builder thread hands its result back through it + with psycopg.connect(partitioned_database) as writer: + _insert_spend_log(writer, "parent-index-lock-owner", "2026-08-15") + builder_thread: Final = threading.Thread( + target=lambda: outcome.append(_build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION)) + ) + builder_thread.start() + try: + assert _wait_for_blocked_ddl(partitioned_database, "%CREATE INDEX%ON ONLY%") + _insert_for(partitioned_database, 3) + finally: + try: + writer.commit() + finally: + builder_thread.join() + assert outcome == [True] + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_inserts_are_never_held_back_while_attach_partition_waits_for_a_reader_of_the_child_index( + partitioned_database: str, +) -> None: + partition: Final = "LiteLLM_SpendLogs_p2026_08" + child_index: Final = CALL_ID_INDEX_DEFINITION.partition_index_name(partition) + assert child_index == "LiteLLM_SpendLogs_p2026_08_litellm_call_id_idx" + with psycopg.connect(partitioned_database, autocommit=True) as conn: + conn.execute( + 'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON ONLY "LiteLLM_SpendLogs" ("litellm_call_id")' + ) + conn.execute( + sql.SQL('CREATE INDEX {} ON {} ("litellm_call_id")').format( + sql.Identifier(CALL_ID_INDEX_DEFINITION.partition_index_name(partition)), sql.Identifier(partition) + ) + ) + + outcome: Final[list[bool]] = [] # mutable-ok: the builder thread hands its result back through it + with psycopg.connect(partitioned_database) as reader: + reader.execute("SET enable_seqscan = off") + reader.execute( + sql.SQL('SELECT count(*) FROM {} WHERE "litellm_call_id" IS NULL').format(sql.Identifier(partition)) + ).fetchone() + reader_pid: Final = reader.execute("SELECT pg_backend_pid()").fetchone()[0] + with psycopg.connect(partitioned_database, autocommit=True) as inspector: + child_lock: Final = inspector.execute( + "SELECT 1 FROM pg_locks WHERE pid = %s AND relation = to_regclass(%s) " + "AND mode = 'AccessShareLock' AND granted", + (reader_pid, f'"{child_index}"'), + ).fetchone() + assert child_lock is not None + builder_thread: Final = threading.Thread( + target=lambda: outcome.append(_build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION)) + ) + builder_thread.start() + try: + assert _wait_for_blocked_ddl(partitioned_database, "%ATTACH PARTITION%") + _insert_for(partitioned_database, 3) + finally: + try: + reader.commit() + finally: + builder_thread.join() + assert outcome == [True] + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_a_parent_index_that_never_gets_its_lock_is_left_for_the_next_index_build( + partitioned_database: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(request_log_indexes, "_DDL_LOCK_ATTEMPTS", 2) + with psycopg.connect(partitioned_database) as writer: + _insert_spend_log(writer, "parent-index-lock-owner", "2026-08-15") + with caplog.at_level("WARNING", logger="litellm_proxy_extras"): + assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is False + assert "leaving it for the next index build" in caplog.text + assert _indexed_table(partitioned_database, CALL_ID_INDEX) is None + writer.commit() + assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is True + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +@requires_db +def test_an_attach_that_never_gets_its_lock_is_left_for_the_next_index_build( + partitioned_database: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + partition: Final = "LiteLLM_SpendLogs_p2026_08" + child_index: Final = CALL_ID_INDEX_DEFINITION.partition_index_name(partition) + assert child_index == "LiteLLM_SpendLogs_p2026_08_litellm_call_id_idx" + with psycopg.connect(partitioned_database, autocommit=True) as conn: + conn.execute( + 'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON ONLY "LiteLLM_SpendLogs" ("litellm_call_id")' + ) + conn.execute( + sql.SQL('CREATE INDEX {} ON {} ("litellm_call_id")').format( + sql.Identifier(child_index), sql.Identifier(partition) + ) + ) + + monkeypatch.setattr(request_log_indexes, "_DDL_LOCK_ATTEMPTS", 2) + with psycopg.connect(partitioned_database) as reader: + reader.execute("SET enable_seqscan = off") + reader.execute( + sql.SQL('SELECT count(*) FROM {} WHERE "litellm_call_id" IS NULL').format(sql.Identifier(partition)) + ).fetchone() + reader_pid: Final = reader.execute("SELECT pg_backend_pid()").fetchone()[0] + with psycopg.connect(partitioned_database, autocommit=True) as inspector: + child_lock: Final = inspector.execute( + "SELECT 1 FROM pg_locks WHERE pid = %s AND relation = to_regclass(%s) " + "AND mode = 'AccessShareLock' AND granted", + (reader_pid, f'"{child_index}"'), + ).fetchone() + assert child_lock is not None + with caplog.at_level("WARNING", logger="litellm_proxy_extras"): + assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is False + assert "Could not get the lock for attaching" in caplog.text + reader.commit() + + assert _build_in_its_own_session(partitioned_database, CALL_ID_INDEX_DEFINITION) is True + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + + +def _build_in_its_own_session(database_url: str, index: RequestLogIndex) -> bool: + with psycopg.connect(database_url, autocommit=True) as builder: + return build_index_on_partitioned_table(builder, "public", index) + + +def _wait_until_the_build_is_waiting(database_url: str) -> None: + deadline: Final = time.monotonic() + 30 + with psycopg.connect(database_url, autocommit=True) as conn: + while time.monotonic() < deadline: + waiting = conn.execute( + "SELECT 1 FROM pg_stat_activity WHERE query LIKE 'CREATE INDEX%' AND wait_event_type IS NOT NULL" + ).fetchone() + if waiting is not None: + return + time.sleep(0.05) + pytest.fail("the partition index build never started waiting on the open write") + + +def _create_index(database_url: str, name: str, table: str, columns: str) -> int: + """Create a plain index by hand, the way an operator's workaround would, and return its oid.""" + with psycopg.connect(database_url, autocommit=True) as conn: + conn.execute( + sql.SQL("CREATE INDEX {} ON {} {}").format(sql.Identifier(name), sql.Identifier(table), sql.SQL(columns)) + ) + return _index_oids(database_url)[name] + + +@requires_db +@pytest.mark.parametrize( + ("definition", "matches"), + ( + (None, False), + ('("litellm_call_id", "api_key")', True), + ('("litellm_call_id") INCLUDE ("api_key")', False), + ), + ids=("missing", "equivalent", "included-column"), +) +def test_equivalent_index_lookup_works_without_postgres_11_catalog_columns( + definition: str | None, matches: bool +) -> None: + schema: Final = f"index_catalog_{uuid.uuid4().hex}" + target: Final = RequestLogIndex("LiteLLM_SpendLogs", "managed_lookup", '("litellm_call_id", "api_key")') + with psycopg.connect(_base_url(), autocommit=True) as conn: + if definition is not None and "INCLUDE" in definition and conn.info.server_version < 110000: + pytest.skip("INCLUDE indexes require PostgreSQL 11") + conn.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + conn.execute( + sql.SQL('CREATE TABLE {}."LiteLLM_SpendLogs" (litellm_call_id text, api_key text)').format( + sql.Identifier(schema) + ) + ) + if definition is not None: + conn.execute( + sql.SQL('CREATE INDEX operator_lookup ON {}."LiteLLM_SpendLogs" {}').format( + sql.Identifier(schema), sql.SQL(definition) + ) + ) + found: Final = request_log_indexes._equivalent_indexes(conn, schema, target.table, target.name, target) + assert tuple((index.name, index.valid) for index in found) == ( + (("operator_lookup", True),) if matches else () + ) + finally: + conn.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +@requires_db +def test_a_valid_index_of_the_same_definition_under_another_name_is_renamed_instead_of_rebuilt( + scratch_database: str, +) -> None: + hand_built: Final = _create_index(scratch_database, "call_id_by_hand", "LiteLLM_SpendLogs", '("litellm_call_id")') + + assert ensure_request_log_indexes(scratch_database, "public") is True + + oids: Final = _index_oids(scratch_database) + assert "call_id_by_hand" not in oids and oids[CALL_ID_INDEX] == hand_built + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + + +@requires_db +def test_a_hand_built_child_index_under_another_name_is_renamed_and_attached(partitioned_database: str) -> None: + partition: Final = "LiteLLM_SpendLogs_p2026_08" + hand_built: Final = _create_index( + partitioned_database, "p2026_08_call_id_by_hand", partition, '("litellm_call_id")' + ) + + assert ensure_request_log_indexes(partitioned_database, "public") is True + + _assert_index_covers_every_partition(partitioned_database, CALL_ID_INDEX, "litellm_call_id_idx") + oids: Final = _index_oids(partitioned_database) + assert "p2026_08_call_id_by_hand" not in oids and oids[f"{partition}_litellm_call_id_idx"] == hand_built + + +@requires_db +def test_an_index_with_another_definition_is_not_taken_for_the_managed_one(scratch_database: str) -> None: + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute(sql.SQL("DROP INDEX {}").format(sql.Identifier(API_KEY_INDEX))) + others: Final = { + "time_then_key": _create_index( + scratch_database, "time_then_key", "LiteLLM_SpendLogs", '("startTime", "api_key")' + ), + "call_id_desc": _create_index( + scratch_database, "call_id_desc", "LiteLLM_SpendLogs", '("litellm_call_id" DESC)' + ), + "call_id_then_key": _create_index( + scratch_database, "call_id_then_key", "LiteLLM_SpendLogs", '("litellm_call_id", "api_key")' + ), + "call_id_pattern": _create_index( + scratch_database, "call_id_pattern", "LiteLLM_SpendLogs", '("litellm_call_id" text_pattern_ops)' + ), + } + + assert ensure_request_log_indexes(scratch_database, "public") is True + + oids: Final = _index_oids(scratch_database) + assert {name: oids[name] for name in others} == others + assert _index_validity(scratch_database, "litellm_call_id_idx") == {CALL_ID_INDEX: True} + assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True} + + +@requires_db +def test_a_valid_partitioned_parent_index_under_another_name_is_renamed_with_its_children_kept( + partitioned_database: str, caplog: pytest.LogCaptureFixture +) -> None: + hand_built: Final = _create_index( + partitioned_database, "call_id_parent_by_hand", "LiteLLM_SpendLogs", '("litellm_call_id")' + ) + children_before: Final = _attached_children(partitioned_database, "call_id_parent_by_hand") + + with caplog.at_level("INFO", logger="litellm_proxy_extras"): + assert ensure_request_log_indexes(partitioned_database, "public") is True + + assert "Building index" not in caplog.text + oids: Final = _index_oids(partitioned_database) + assert "call_id_parent_by_hand" not in oids and oids[CALL_ID_INDEX] == hand_built + assert _attached_children(partitioned_database, CALL_ID_INDEX) == children_before + assert _index_validity(partitioned_database, "litellm_call_id_idx")[CALL_ID_INDEX] is True + + +@requires_db +def test_a_second_copy_of_a_managed_index_is_reported_with_its_drop_statement_and_left_in_place( + scratch_database: str, caplog: pytest.LogCaptureFixture +) -> None: + assert ensure_request_log_indexes(scratch_database, "public") is True + copy: Final = _create_index(scratch_database, "call_id_copy", "LiteLLM_SpendLogs", '("litellm_call_id")') + + with caplog.at_level("WARNING", logger="litellm_proxy_extras"): + assert ensure_request_log_indexes(scratch_database, "public") is True + + assert 'remove it with: DROP INDEX CONCURRENTLY "public"."call_id_copy"' in caplog.text + assert _index_oids(scratch_database)["call_id_copy"] == copy diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py index 99e494fccd5..f753ddce3c1 100644 --- a/tests/test_litellm/proxy/db/test_prisma_client.py +++ b/tests/test_litellm/proxy/db/test_prisma_client.py @@ -3,6 +3,7 @@ import os import signal import sys import urllib.parse +from collections.abc import Callable from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -454,3 +455,21 @@ def test_db_push_without_the_prisma_runner_fails_the_migration_instead_of_crashi assert PrismaManager.setup_database(use_migrate=False) is False assert fake_prisma_cli.calls == [] + + +@pytest.mark.parametrize( + ("run", "outcome"), + ( + (PrismaManager.build_request_log_indexes, False), + (PrismaManager.start_request_log_index_build, None), + ), + ids=("wait-for-the-build", "start-the-build"), +) +def test_without_proxy_extras_the_index_build_reports_failure_instead_of_raising( + monkeypatch: pytest.MonkeyPatch, run: Callable[[], bool | None], outcome: bool | None +) -> None: + """The migration job exits non-zero and a serving proxy keeps booting when the extras + package that owns the index build is not installed.""" + monkeypatch.setitem(sys.modules, "litellm_proxy_extras.utils", None) + + assert run() is outcome diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index c451191605f..93b2d189b90 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -2,14 +2,14 @@ import inspect import os from contextlib import nullcontext from pathlib import Path +from queue import SimpleQueue from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import click import fastapi import pytest - - import builtins import types import urllib.parse as urlparse @@ -19,6 +19,7 @@ import yaml from uvicorn.config import LOOP_FACTORIES from uvicorn.importer import import_from_string +from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server @@ -1924,19 +1925,29 @@ class TestQueryEngineReaperWiring: class TestRunServerDbSetup: """Tests for run_server's prisma setup_database behavior.""" + @pytest.fixture + def mock_setup_database(self) -> MagicMock: + return MagicMock() + + @pytest.fixture + def database_startup(self, mock_setup_database: MagicMock) -> proxy_cli._DatabaseStartupDependencies: + return proxy_cli._DatabaseStartupDependencies( + setup=mock_setup_database, build=lambda: True, start=lambda: None + ) + @patch("subprocess.run") @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") def test_use_prisma_db_push_flag_behavior( self, - mock_should_update_schema, - mock_check_schema_diff, - mock_setup_database, - mock_atexit_register, - mock_subprocess_run, - ): + mock_should_update_schema: MagicMock, + mock_check_schema_diff: MagicMock, + mock_atexit_register: MagicMock, + mock_subprocess_run: MagicMock, + mock_setup_database: MagicMock, + database_startup: proxy_cli._DatabaseStartupDependencies, + ) -> None: """Test that use_prisma_db_push flag correctly controls PrismaManager.setup_database use_migrate parameter""" from litellm.proxy.proxy_cli import run_server @@ -1946,19 +1957,17 @@ class TestRunServerDbSetup: # Mock should_update_prisma_schema to return True (so setup_database gets called) mock_should_update_schema.return_value = True - mock_proxy_module = MagicMock( + mock_proxy_module: Final = MagicMock( app=MagicMock(), ProxyConfig=MagicMock(), KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") + clean_env: Final = { + **{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}, + "DATABASE_URL": "postgresql://test:test@localhost:5432/test", } - clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" with ( patch.dict(os.environ, clean_env, clear=True), @@ -1985,10 +1994,8 @@ class TestRunServerDbSetup: # Test 1: Without --use_prisma_db_push flag (default behavior) # use_prisma_db_push should be False (default), so use_migrate should be True - run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) - mock_setup_database.assert_called_with( - use_migrate=True, use_v2_resolver=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup) + mock_setup_database.assert_called_with(use_migrate=True, use_v2_resolver=False) # Reset mocks mock_setup_database.reset_mock() @@ -2000,43 +2007,44 @@ class TestRunServerDbSetup: run_server.main( ["--local", "--skip_server_startup", "--use_prisma_db_push"], standalone_mode=False, + obj=database_startup, ) - mock_setup_database.assert_called_with( - use_migrate=False, use_v2_resolver=False - ) + mock_setup_database.assert_called_with(use_migrate=False, use_v2_resolver=False) @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.check_migration.check_prisma_schema_diff" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above def test_migrations_run_when_the_prisma_cli_is_not_on_path( self, - mock_should_update_schema, - mock_check_schema_diff, - mock_setup_database, - mock_atexit_register, - tmp_path, - capsys, - ): + mock_should_update_schema: MagicMock, + mock_check_schema_diff: MagicMock, + mock_atexit_register: MagicMock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + mock_setup_database: MagicMock, + database_startup: proxy_cli._DatabaseStartupDependencies, + ) -> None: from litellm.proxy.proxy_cli import run_server mock_should_update_schema.return_value = True - empty_bin = tmp_path / "emptybin" + empty_bin: Final = tmp_path / "emptybin" empty_bin.mkdir() - mock_proxy_module = MagicMock( + mock_proxy_module: Final = MagicMock( app=MagicMock(), ProxyConfig=MagicMock(), KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") + clean_env: Final = { + **{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}, + "DATABASE_URL": "postgresql://test:test@localhost:5432/test", } - clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" clean_env["PATH"] = str(empty_bin) with ( @@ -2058,12 +2066,10 @@ class TestRunServerDbSetup: "port": 8000, } - run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup) assert "prisma CLI is neither on PATH" not in capsys.readouterr().out - mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=False - ) + mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=False) @pytest.mark.parametrize( ("arguments", "env", "exits"), @@ -2079,21 +2085,21 @@ class TestRunServerDbSetup: ) @patch("subprocess.run") @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") def test_startup_fails_when_db_setup_fails( self, - mock_should_update_schema, - mock_check_schema_diff, - mock_setup_database, - mock_atexit_register, - mock_subprocess_run, - arguments, - env, - exits, - capsys, - ): + mock_should_update_schema: MagicMock, + mock_check_schema_diff: MagicMock, + mock_atexit_register: MagicMock, + mock_subprocess_run: MagicMock, + arguments: tuple[str, ...], + env: dict[str, str], + exits: bool, + capsys: pytest.CaptureFixture[str], + mock_setup_database: MagicMock, + database_startup: proxy_cli._DatabaseStartupDependencies, + ) -> None: """A failed PrismaManager.setup_database exits 1 by default; only an explicit --no-enforce_prisma_migration_check or ENFORCE_PRISMA_MIGRATION_CHECK=false keeps the proxy booting""" from litellm.proxy.proxy_cli import run_server @@ -2102,20 +2108,22 @@ class TestRunServerDbSetup: mock_should_update_schema.return_value = True mock_setup_database.return_value = False - mock_proxy_module = MagicMock( + mock_proxy_module: Final = MagicMock( app=MagicMock(), ProxyConfig=MagicMock(), KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL", "ENFORCE_PRISMA_MIGRATION_CHECK") + clean_env: Final = { + **{ + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL", "ENFORCE_PRISMA_MIGRATION_CHECK") + }, + "DATABASE_URL": "postgresql://test:test@localhost:5432/test", } - clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" - outcome = pytest.raises(SystemExit) if exits else nullcontext() + outcome: Final = pytest.raises(SystemExit) if exits else nullcontext() with ( patch.dict(os.environ, {**clean_env, **env}, clear=True), @@ -2136,27 +2144,27 @@ class TestRunServerDbSetup: "host": "localhost", "port": 8000, } - run_server.main(["--local", "--skip_server_startup", *arguments], standalone_mode=False) + run_server.main( + ["--local", "--skip_server_startup", *arguments], standalone_mode=False, obj=database_startup + ) assert (exc_info is not None and exc_info.value.code == 1) is exits assert ("Database migration failed but continuing startup" in capsys.readouterr().out) is not exits - mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=False - ) + mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=False) @patch("subprocess.run") @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") def test_startup_exits_on_non_postgres_database_url( self, - mock_should_update_schema, - mock_check_schema_diff, - mock_setup_database, - mock_atexit_register, - mock_subprocess_run, - ): + mock_should_update_schema: MagicMock, + mock_check_schema_diff: MagicMock, + mock_atexit_register: MagicMock, + mock_subprocess_run: MagicMock, + mock_setup_database: MagicMock, + database_startup: proxy_cli._DatabaseStartupDependencies, + ) -> None: """A sqlite DATABASE_URL must exit immediately, before any prisma call, instead of stalling on a migration against the postgresql-only schema.""" from litellm.proxy.proxy_cli import run_server @@ -2164,19 +2172,17 @@ class TestRunServerDbSetup: mock_subprocess_run.return_value = MagicMock(returncode=0) mock_should_update_schema.return_value = True - mock_proxy_module = MagicMock( + mock_proxy_module: Final = MagicMock( app=MagicMock(), ProxyConfig=MagicMock(), KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") + clean_env: Final = { + **{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}, + "DATABASE_URL": "sqlite:///data/litellm.db", } - clean_env["DATABASE_URL"] = "sqlite:///data/litellm.db" with ( patch.dict(os.environ, clean_env, clear=True), @@ -2189,30 +2195,29 @@ class TestRunServerDbSetup: ), ): with pytest.raises(SystemExit) as exc_info: - run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup) assert exc_info.value.code == 1 mock_setup_database.assert_not_called() @patch("subprocess.run") @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") def test_v2_migration_resolver_opts_in_via_env_var( self, - mock_should_update_schema, - mock_check_schema_diff, - mock_setup_database, - mock_atexit_register, - mock_subprocess_run, - ): - """USE_V2_MIGRATION_RESOLVER must select the v2 resolver. + mock_should_update_schema: MagicMock, + mock_check_schema_diff: MagicMock, + mock_atexit_register: MagicMock, + mock_subprocess_run: MagicMock, + capsys: pytest.CaptureFixture[str], + mock_setup_database: MagicMock, + database_startup: proxy_cli._DatabaseStartupDependencies, + ) -> None: + """USE_V2_MIGRATION_RESOLVER=true must select the v2 resolver. The Helm migrations Job runs `python litellm/proxy/prisma_migration.py`, - which calls run_server with a fixed argv, so a deployment has no way to - pass --use_v2_migration_resolver and an env var is the only route in. + which calls run_server with a fixed argv, so a deployment reaches the + resolver through the env var rather than a CLI flag. """ from litellm.proxy.proxy_cli import run_server @@ -2220,20 +2225,18 @@ class TestRunServerDbSetup: mock_should_update_schema.return_value = True mock_setup_database.return_value = True - mock_proxy_module = MagicMock( + mock_proxy_module: Final = MagicMock( app=MagicMock(), ProxyConfig=MagicMock(), KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") + clean_env: Final = { + **{k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}, + "DATABASE_URL": "postgresql://test:test@localhost:5432/test", + "USE_V2_MIGRATION_RESOLVER": "true", } - clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" - clean_env["USE_V2_MIGRATION_RESOLVER"] = "true" with ( patch.dict(os.environ, clean_env, clear=True), @@ -2245,13 +2248,56 @@ class TestRunServerDbSetup: }, ), ): - run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False, obj=database_startup) - mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=True + mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True) + + @pytest.mark.parametrize( + ("skip_server_startup", "migrated", "build_succeeds", "calls_expected", "succeeds"), + ( + (True, True, False, ("setup", "build"), True), + (True, True, True, ("setup", "build"), True), + (False, True, False, ("setup", "start"), True), + (False, False, False, ("setup",), False), + (True, False, False, ("setup",), False), + ), + ids=( + "failed-build", + "migration-job", + "serving-proxy", + "serving-proxy-whose-migrations-failed", + "failed-migration-job", + ), + ) + def test_the_migration_job_waits_for_the_index_build_and_a_serving_proxy_starts_it_in_the_background( + self, + skip_server_startup: bool, + migrated: bool, + build_succeeds: bool, + calls_expected: tuple[str, ...], + succeeds: bool, + ) -> None: + calls: Final[SimpleQueue[str]] = SimpleQueue() + + def setup() -> bool: + calls.put("setup") + return migrated + + def build() -> bool: + calls.put("build") + return build_succeeds + + def start() -> None: + calls.put("start") + + assert ( + proxy_cli._setup_database_with_request_log_indexes( + skip_server_startup, setup=setup, build=build, start=start + ) + is succeeds ) + assert tuple(calls.get_nowait() for _ in calls_expected) == calls_expected + assert calls.empty() # --- Module-level helpers for worker startup hook tests --- diff --git a/tests/test_litellm/test_check_migrations_no_data_rewrites.py b/tests/test_litellm/test_check_migrations_no_data_rewrites.py index c5d3cdd9073..a2e9bb6887d 100644 --- a/tests/test_litellm/test_check_migrations_no_data_rewrites.py +++ b/tests/test_litellm/test_check_migrations_no_data_rewrites.py @@ -9,6 +9,9 @@ an `UPDATE` hidden in the `DO $$ ... $$` block this repo uses for conditional DD import importlib.util import sys from pathlib import Path +from typing import Final + +import pytest _CHECKER_PATH = Path(__file__).resolve().parents[1] / "code_coverage_tests" / "check_migrations_no_data_rewrites.py" _SPEC = importlib.util.spec_from_file_location("check_migrations_no_data_rewrites", _CHECKER_PATH) @@ -205,6 +208,96 @@ class TestDefaultedColumnsOnRequestLogTables: assert 'ADD COLUMN ... DEFAULT on "LiteLLM_SpendLogs" rewrites existing rows at boot' in rendered +class TestIndexesOnLogTables: + """Every CREATE INDEX on a request-log table is rejected: a plain one blocks writes for + the whole build and a concurrent one fails on a partitioned parent, so the migration job + (litellm_proxy_extras/request_log_indexes.py) builds those instead.""" + + def test_the_original_spend_log_index_statement_is_flagged(self, tmp_path: Path) -> None: + sql: Final = ( + "-- CreateIndex\n" + 'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" ' + 'ON "LiteLLM_SpendLogs"("api_key", "startTime");' + ) + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_the_original_concurrent_call_id_index_statement_is_flagged(self, tmp_path: Path) -> None: + sql: Final = ( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" ' + 'ON "LiteLLM_SpendLogs"("litellm_call_id");' + ) + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_unique_index_with_if_not_exists_on_error_logs_is_flagged(self, tmp_path: Path) -> None: + sql: Final = 'CREATE UNIQUE INDEX IF NOT EXISTS "ix" ON "LiteLLM_ErrorLogs" ("request_id");' + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_ErrorLogs"',) + + def test_a_unique_concurrent_index_on_error_logs_is_flagged(self, tmp_path: Path) -> None: + sql: Final = 'CREATE UNIQUE INDEX CONCURRENTLY "ix" ON "LiteLLM_ErrorLogs" ("request_id");' + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_ErrorLogs"',) + + def test_lowercase_schema_qualified_and_only_forms_are_flagged(self, tmp_path: Path) -> None: + sql: Final = ( + 'create index on "public"."LiteLLM_SpendLogs" ("api_key");\n' + 'CREATE INDEX "ix" ON ONLY "LiteLLM_SpendLogs" ("api_key");\n' + 'CREATE INDEX CONCURRENTLY "iy" ON "public"."LiteLLM_SpendLogs" ("api_key");' + ) + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) * 3 + + def test_a_comment_between_on_and_the_table_is_flagged(self, tmp_path: Path) -> None: + sql: Final = 'CREATE INDEX CONCURRENTLY "ix" ON /* table */ "LiteLLM_SpendLogs" ("api_key");' + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_a_concurrent_index_with_comments_and_line_breaks_is_flagged(self, tmp_path: Path) -> None: + sql: Final = ( + "-- CreateIndex\n" + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "ix"\n' + ' ON "LiteLLM_SpendLogs" /* partitioned in some deployments */\n' + ' ("api_key", "startTime");\n' + ) + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_indexes_on_a_non_log_table_pass_concurrent_or_not(self, tmp_path: Path) -> None: + sql: Final = ( + 'CREATE INDEX "ix" ON "LiteLLM_VerificationToken" ("token");\n' + 'CREATE INDEX CONCURRENTLY "iy" ON "LiteLLM_VerificationToken" ("token");' + ) + assert _keywords(tmp_path, sql) == () + + def test_an_index_run_by_execute_is_flagged(self, tmp_path: Path) -> None: + sql: Final = 'DO $$ BEGIN EXECUTE \'CREATE INDEX "ix" ON "LiteLLM_SpendLogs" ("api_key")\'; END $$;' + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_a_marker_does_not_exempt_the_index(self, tmp_path: Path) -> None: + sql: Final = ( + '-- data-migration-ok: table is empty at this point\nCREATE INDEX "ix" ON "LiteLLM_SpendLogs" ("api_key");' + ) + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_a_marker_on_a_rewrite_still_leaves_the_index_below_it_flagged(self, tmp_path: Path) -> None: + sql: Final = ( + "-- data-migration-ok: one row\n" + 'UPDATE "LiteLLM_SpendLogs" SET "api_key" = \'k\' WHERE "request_id" = \'r\';\n' + 'CREATE INDEX CONCURRENTLY "ix" ON "LiteLLM_SpendLogs" ("api_key");' + ) + assert _keywords(tmp_path, sql) == ('CREATE INDEX on "LiteLLM_SpendLogs"',) + + def test_render_points_at_the_migration_job_index_list(self, tmp_path: Path) -> None: + sql: Final = 'CREATE INDEX CONCURRENTLY "ix" ON "public"."LiteLLM_SpendLogs" ("api_key");' + rendered: Final = _scan(tmp_path, sql)[0].render() + assert "20260101000000_fixture/migration.sql:1" in rendered + assert "blocks writes until the build finishes, or fails on a partitioned table" in rendered + assert "REQUEST_LOG_INDEXES in litellm_proxy_extras/request_log_indexes.py" in rendered + + @pytest.mark.parametrize( + "name", + ("20260823000000_add_spend_logs_api_key_starttime_index", "20260831120001_spend_logs_litellm_call_id_index"), + ) + def test_the_inert_index_migrations_scan_clean_without_a_grandfather(self, name: str) -> None: + assert checker.scan_migration(checker.MIGRATIONS_DIR / name) == () + assert name not in checker.GRANDFATHERED + + class TestInsert: def test_insert_values_is_bounded_and_passes(self, tmp_path): assert _keywords(tmp_path, "INSERT INTO \"Foo\" (\"id\") VALUES ('a'), ('b');") == () diff --git a/uv.lock b/uv.lock index ea47b653d30..be10806c4ce 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-10-20T22:36:52.444783Z" +exclude-newer = "2026-11-07T05:48:32.556148868Z" exclude-newer-span = "P3D" [options.exclude-newer-package] @@ -1660,60 +1660,70 @@ wheels = [ [[package]] name = "ddtrace" -version = "4.11.0" +version = "4.15.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bytecode" }, + { name = "ddtrace-internal" }, { name = "envier" }, { name = "opentelemetry-api" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/51/a628f0177274bab5b67c93a1558fd222babb15286a427e4e8c1d65a265b2/ddtrace-4.11.0.tar.gz", hash = "sha256:260c5b46e80565f4fd08cec2650f707627fb57cd6f8951a2d8c9e6a02b490074", size = 2422158, upload-time = "2026-07-10T08:57:53.396Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/94/f0f005257dc49a7062c6fed98a6dbcca7691e9d1db5c64e8a2d14f204cac/ddtrace-4.15.6.tar.gz", hash = "sha256:2914f646d4ae338c644b814c4ad80c963e7aadfb39d1e67b0367a15d5c4bc0db", size = 2655968, upload-time = "2026-10-08T09:46:58.392Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/16/1f/05dc12819ca3a3000028a522cab0025c5dd969e97dc472ac38a7d2f1ea71/ddtrace-4.11.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:0bc55e48da62b6a287b36e01f5876489023178c09b60e8e01608e2e895ccdc4b", size = 7103767, upload-time = "2026-07-10T08:55:10.348Z" }, - { url = "https://files.pythonhosted.org/packages/e8/8a/efcea448bc15b4d95c48c9b476f2ddc5862a53cd4baf58fe50c64763f9d8/ddtrace-4.11.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5df47873c625993e81fabd8f6ae2ab61f66c6805cfd1a8c551a509d5b5d91062", size = 7447104, upload-time = "2026-07-10T08:55:15.794Z" }, - { url = "https://files.pythonhosted.org/packages/39/28/89aa239e47d28900ab394a0b8e0a5549cab631c0e5977016ad9d578eaee7/ddtrace-4.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e857428cc86fe541f5b3768d7d8331e2f1cbde1a2492deedff4619109cbfd5fc", size = 8511839, upload-time = "2026-07-10T08:55:17.609Z" }, - { url = "https://files.pythonhosted.org/packages/30/43/37a5475ade911abae592e121e852922745d842e16294c549cba4ad0f3dbe/ddtrace-4.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc60575f8f47af58e6523f8a802d1f92a4912aa7d4ba71024958421044cba61a", size = 8733381, upload-time = "2026-07-10T08:55:19.66Z" }, - { url = "https://files.pythonhosted.org/packages/10/df/d61f71236a405480c8809522219b184cc486056c32f9cc8bf07cc91e2be9/ddtrace-4.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d65ab60bfb0b2edde6c564bd1f97b99a8ea9bb082c93b265488ba214c606d240", size = 9513521, upload-time = "2026-07-10T08:55:21.921Z" }, - { url = "https://files.pythonhosted.org/packages/79/4f/bff08af986df419179b86fc9c73732836531c2d08ab8dcfe982a6f2e8cb8/ddtrace-4.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7ebfdee1342b5b1cdad8505e040bbfe5bdf6976e820803eaa4a1814417d6bb0e", size = 9788008, upload-time = "2026-07-10T08:55:24.111Z" }, - { url = "https://files.pythonhosted.org/packages/af/b8/2b1fd5b0374edf33ad31adb746d1d1358a44c39a45b1c2a0c4b09651d81e/ddtrace-4.11.0-cp310-cp310-win32.whl", hash = "sha256:6247a431ae3a2622a0e835357a644b1bac81719e28f21748fd7d4034c79c3ed7", size = 5739013, upload-time = "2026-07-10T08:55:27.028Z" }, - { url = "https://files.pythonhosted.org/packages/f2/f5/9bdff59bdce1f80592e70633dc26678c762fff79188c9fb6004c06967708/ddtrace-4.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:9197110982756a97243906ffce7b98e6007c8258a6bbc22451d7b26e8f1b0103", size = 6326099, upload-time = "2026-07-10T08:55:28.962Z" }, - { url = "https://files.pythonhosted.org/packages/76/36/ad0abcc97b63d821ada82cc7452c3070a71f9546f0aa81b50eda1f8ffd41/ddtrace-4.11.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b4074e538e708b0e9d7f812be15db306abbe3a6cb70046f9f786d66daa862344", size = 7105089, upload-time = "2026-07-10T08:55:31.006Z" }, - { url = "https://files.pythonhosted.org/packages/a6/3a/c0f8f70883a70ec9d4050900823ff7b52ad54dc6b2a3d29561727825410f/ddtrace-4.11.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:05f701229233ad8a4ad5d52b8ff2bfd6e2f99601a3bc04367615cd1ce4a01a80", size = 7448237, upload-time = "2026-07-10T08:55:32.962Z" }, - { url = "https://files.pythonhosted.org/packages/4e/46/18a42afab21cd718467e7d7ae3e80898569d9969d731b771b5a8ada10294/ddtrace-4.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c3a557891e91a443e6a3a9b7c31dc96b9963acada0986dab5a3d65507efa16a1", size = 8516433, upload-time = "2026-07-10T08:55:35.011Z" }, - { url = "https://files.pythonhosted.org/packages/da/b2/6367a75518f843872e13c062ac3c329f880b2f4d62ff4de0624bdcf64719/ddtrace-4.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0328bfeb4a6bdc00168a4184aea670eedc073734f6fbc6ca1abfeb6547415d2", size = 8740643, upload-time = "2026-07-10T08:55:37.743Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e8/04f1d7ee4737444d45f81545ce21b01bd9d653a994997ab1e7e04b6ee947/ddtrace-4.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad6f17111746568a58da89b6db5e5d18eb07db7b8ce3be3396631f37391cff50", size = 9518626, upload-time = "2026-07-10T08:55:40.248Z" }, - { url = "https://files.pythonhosted.org/packages/ed/8b/314ba8a81e562a31655328061afb30d26d46ba04aca4dabd7da85c93a9e7/ddtrace-4.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eb30dd327789ec1851bc42194c8db0b93a7ad708c2963d694e3622b684a085f7", size = 9795392, upload-time = "2026-07-10T08:55:42.609Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6e/a580d53c05a771bc53d33a88174e983146fdf879fe5d4f52e51ceb56ee99/ddtrace-4.11.0-cp311-cp311-win32.whl", hash = "sha256:dfe73130778ca22652ae398fc77de3643d50687deb127bf4cd2566b10778125a", size = 5737732, upload-time = "2026-07-10T08:55:45.605Z" }, - { url = "https://files.pythonhosted.org/packages/12/83/094f5b88690f7801129b0d52a7b6a58a95b34ecbfcb337555d323c639faa/ddtrace-4.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:1241f38d52b40b516d620ed68ecd04afbc715f2c1c9f3ddae8a7b86ecf848c91", size = 6329473, upload-time = "2026-07-10T08:55:47.603Z" }, - { url = "https://files.pythonhosted.org/packages/60/72/036d11ceca6fbdc4ef95a1aba210cb3fc72bc912703cba49cf2bc8b610c5/ddtrace-4.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:3013b67e1a172eb0edd8c37559b81760a9db205a671541925ebd0127f496409b", size = 5990873, upload-time = "2026-07-10T08:55:49.705Z" }, - { url = "https://files.pythonhosted.org/packages/f7/e9/e50295e520f7d14ce2b5d900f7a2fe8f096d8f90e9040f8d7f43108b72ce/ddtrace-4.11.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d0eadcf7df3143bf0611f09a351040e475b5df7f512410698cd481a076bca056", size = 7101986, upload-time = "2026-07-10T08:55:56.51Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c6/e8d13546a4b2cd72880d8b0823dd487956d9eb291473d7d03b6e742e5bd9/ddtrace-4.11.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:57eb0616dc5c8ff04b5c560ba6d3def69d2bc618e69ce65cc015f103fc68649e", size = 7455362, upload-time = "2026-07-10T08:55:58.664Z" }, - { url = "https://files.pythonhosted.org/packages/50/c6/2c22d71290df36b87393794d6d803ec8ea96ee54d42286836a7ddf0dbae3/ddtrace-4.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c311109d1e0f3482b7a44126c855eeb796b186f01604ce74c54800aa5892863d", size = 8502274, upload-time = "2026-07-10T08:56:00.869Z" }, - { url = "https://files.pythonhosted.org/packages/19/84/5afef2c9696b289afa1c608e862930c0e447489776a0133da9f2345edd27/ddtrace-4.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:695ec915640072e2917b13d317fe1696086a38924ba02cbe1dcb3918ffb0fa65", size = 8730138, upload-time = "2026-07-10T08:56:03.711Z" }, - { url = "https://files.pythonhosted.org/packages/12/68/a2aff5999fa2b1053b3f50cd78297cbbe7a427f63388fb5f6c500dfb859a/ddtrace-4.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:355a288756a7a5d6b73221d28ba4536cb0271d2ea7909adab3e2ee37d62a569f", size = 9507981, upload-time = "2026-07-10T08:56:06.317Z" }, - { url = "https://files.pythonhosted.org/packages/79/12/53bb54b3451f85dbf3c752c2d60b22f672fd55bb08d7495613a70dc87102/ddtrace-4.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c19888af09f6ad3340f2935ea5c9eb80d9298141e196fd33e544369920aefba", size = 9793730, upload-time = "2026-07-10T08:56:08.893Z" }, - { url = "https://files.pythonhosted.org/packages/9c/7e/d9a52dd0d877ebdce5eb105d9f3d4e3579391ee0106c8335d59aa97006dd/ddtrace-4.11.0-cp312-cp312-win32.whl", hash = "sha256:cd524b299bd15ed14192cb041a35e992003e0d3a9f8831062f9f28bcfacb63e2", size = 5734745, upload-time = "2026-07-10T08:56:11.764Z" }, - { url = "https://files.pythonhosted.org/packages/6d/da/bb7c609ebc368cabb01db38d7022f42a39d30207f484a5e18a306bd9b333/ddtrace-4.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:af663120d3b4116965bebbd79ce8e73a85644816a0917999e8164b6204678cb6", size = 6317182, upload-time = "2026-07-10T08:56:14.156Z" }, - { url = "https://files.pythonhosted.org/packages/30/f3/e3cf49ce0a4c377fdea8203b127984e26ac2c9cad0521fab27d61371d568/ddtrace-4.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:f0b50a1739573318b0a81fb56a8b22ed6ae6dac017b89400cbc76c17b439d9ed", size = 5980198, upload-time = "2026-07-10T08:56:16.458Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f8/89f2b9cb3d413c55a3d1f19a90fda76ec45abdc0ce13a7f6d4aaf10bed1a/ddtrace-4.11.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:16b9f06c841858debb3de2c4a7ce6acbd6967eaa6ce264dbd0e2a58e376b8583", size = 7094539, upload-time = "2026-07-10T08:56:18.943Z" }, - { url = "https://files.pythonhosted.org/packages/ed/98/f262ad401cdd1a8a2394299d12f31bcc037d0e2dc61d3a843f1c855c293f/ddtrace-4.11.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:1cb352d7df16b864f258d4602addf23b8d29d2d416ccc1d0af8d88c614f70a01", size = 7448347, upload-time = "2026-07-10T08:56:21.359Z" }, - { url = "https://files.pythonhosted.org/packages/1f/17/c9123fc263b89934cc1c831b81d90af299d28512740abf008fd963bd006e/ddtrace-4.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:54f4cf656aa452bc37b62fa7939c5bddca751ffd4d315b3c925236d4794767ea", size = 8497865, upload-time = "2026-07-10T08:56:24.143Z" }, - { url = "https://files.pythonhosted.org/packages/39/31/00d529b9ca80c77bf3c6aa7370615e2186fc9ead573ef56cef345e6a4dbf/ddtrace-4.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:776d99ccb01abecf7dbb4910f4a570b6ab625ed47e00bb6c0159d88d4e2ef1c8", size = 8721481, upload-time = "2026-07-10T08:56:26.983Z" }, - { url = "https://files.pythonhosted.org/packages/41/56/759eab73938820054faa6db6eda607753d821b6322ce7679a2b9f728f8c7/ddtrace-4.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5533ce94e50412565a36ce0c8fc6ff6f7490e69b4ecff10c550482e49d5d065e", size = 9505774, upload-time = "2026-07-10T08:56:29.901Z" }, - { url = "https://files.pythonhosted.org/packages/5d/a4/6ab67386244c08340c1c30affffc644b14095e59eb8d2e8d8ee1d4d46d84/ddtrace-4.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:81d147bd3d6fa067f2c72c7e32aba90d5d61ed379850896f1cc7fd1ea3437bb1", size = 9787192, upload-time = "2026-07-10T08:56:32.824Z" }, - { url = "https://files.pythonhosted.org/packages/e3/8d/44e074f927b35007e8bf3dc2b5aeb5ba5e4c97e76d4b8e994aaabddfd648/ddtrace-4.11.0-cp313-cp313-win32.whl", hash = "sha256:bcf1d3ba8c3771b8c75f0b9ecd9038ae0a44c8bb0c01d21458c405a29341518a", size = 5732179, upload-time = "2026-07-10T08:56:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/7e/ec/ff034010a1fd598863a15606aa7867ee6abaf4a5989b1a22e61753229432/ddtrace-4.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:d465b9f6f049fc32278cbfd74e1e593b0c3460aacb622bef0cee71e50ffc4b05", size = 6314643, upload-time = "2026-07-10T08:56:38.565Z" }, - { url = "https://files.pythonhosted.org/packages/bc/dd/ad7da7ca071c76815b16a0523875c0c061902cfd8e9ae6595577464e923e/ddtrace-4.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:00cc0fbfa6b8741477528c3fe7bc70a4a5ea439221f188e14da6907f03ed0125", size = 5976789, upload-time = "2026-07-10T08:56:41.172Z" }, - { url = "https://files.pythonhosted.org/packages/12/1b/37b9a84cc4a95d74d4358a0f78346376bc03625b6a9fa0d568a1587665ca/ddtrace-4.11.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e6f1ca84237e9f52fd71e3458262cfc407dd43337a56e4a6ba9beea88f0958c5", size = 7099136, upload-time = "2026-07-10T08:56:43.766Z" }, - { url = "https://files.pythonhosted.org/packages/f1/42/f227eeb9c5bbb519c67fe11a5326b531f4c1bbd90257e74ab8b72b5ded75/ddtrace-4.11.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:c3ddbf9a5a0cd51b5ab9bca898c85cfce73072054b9f5b577164a95065cb8331", size = 7452151, upload-time = "2026-07-10T08:56:46.822Z" }, - { url = "https://files.pythonhosted.org/packages/eb/c4/35fbc0e8ef585d93f82dadae38246eb3643621f08efe11ca0c20e77e7d42/ddtrace-4.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7c435b708f4956208141c22d941ed60a1551813353f6c1cfc1a5283ec3d80cc5", size = 8507170, upload-time = "2026-07-10T08:56:49.646Z" }, - { url = "https://files.pythonhosted.org/packages/70/99/a9681a53e9b0557497b24a4e0b1495acf2bcb74368e86129e549d6249a4a/ddtrace-4.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:26b3fd7ee97cf7fa1296eecacd393bbbef343cf8bb2d0b1ee39d40cb1d4818fa", size = 8725097, upload-time = "2026-07-10T08:56:53.106Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c6/b18f02120c34abf55e28bc180e872ddbe27614559c01bbcfe5be35b5494a/ddtrace-4.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8d23d66ba9525fd9b9a61c0db8c43e64461066b1d49c75c85f639ea65776367b", size = 9516895, upload-time = "2026-07-10T08:56:56.513Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ed/dc73e8317c229300b0ff8513f903d16ff140eede8084b3b6f76c8646a548/ddtrace-4.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:85e081e2295a77c9b38e5792a097f1df8169e1dd9b3339f820d00183e10e8b1b", size = 9792739, upload-time = "2026-07-10T08:57:00.333Z" }, - { url = "https://files.pythonhosted.org/packages/7c/8e/b97984d9f43e2f36f5317034711140c3bd1fbb5d96880b1c11b9e1e8c3d3/ddtrace-4.11.0-cp314-cp314-win32.whl", hash = "sha256:c267fe4efceb71019f5e6833fdb4fbdff18a51641bdca9fceabbe579984461e4", size = 5831072, upload-time = "2026-07-10T08:57:03.578Z" }, - { url = "https://files.pythonhosted.org/packages/e8/6a/7fb18925588f645d6d777c90d388f9a640adf7b986be7457943717b74170/ddtrace-4.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bebf3cb12ac51c422ad1ed74873746b8398cce845075b234115c2f518a7a652", size = 6455126, upload-time = "2026-07-10T08:57:06.362Z" }, - { url = "https://files.pythonhosted.org/packages/a0/b2/44d7f07ba94c70bd1827669d446a2e979f7efd6e4a409c469a59684d9a7b/ddtrace-4.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:f596cb2da582600e091c8174ab4f8e44bf4f05c50c03cc450e0378c516e0c6e3", size = 6130096, upload-time = "2026-07-10T08:57:09.413Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c8/6b8de8b78503f54a6c19f5b4c9430825976f1a2ef1b8342f21ac47c70e22/ddtrace-4.15.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:68c18fa37cd718f2360c980bbd2928a5c2bf502a173e61a83f25e42321d5b4b8", size = 7998372, upload-time = "2026-10-08T09:44:43.566Z" }, + { url = "https://files.pythonhosted.org/packages/67/30/b5ec13494b638453c568ced825b863c086fc7237528aa30498b4fcb6928a/ddtrace-4.15.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7d7696cf05e1675847303701c0a2f7733014f1b2cb5938ed2038e27fa32a18b2", size = 8382818, upload-time = "2026-10-08T09:44:45.73Z" }, + { url = "https://files.pythonhosted.org/packages/fa/dc/57c5d150061cf27935a2e630ca5334f3fa815d82f0585ba694821f793445/ddtrace-4.15.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:66647b3c9673468e35fe0294d25015ef6f535bb540a9ce63c76c95ff2679e059", size = 9544001, upload-time = "2026-10-08T09:44:47.366Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/6651a225df4438761ddd45c21f624d453aca1508d88ed7fd4c880a3a454a/ddtrace-4.15.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4fd260a1339a2df4867f5b0f7ff9111a0f23ba86d6b402e4c3dfe2f71853cd17", size = 9794825, upload-time = "2026-10-08T09:44:49.468Z" }, + { url = "https://files.pythonhosted.org/packages/e4/8d/ffd12a1fa866068c397a1fbbebaebcd6e40f21e618650e95d93860afef44/ddtrace-4.15.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:95a04ff00e401caef808689ae6bd70a9fc92d3572647f9f1bbd43dfacfa053f6", size = 10533745, upload-time = "2026-10-08T09:44:51.57Z" }, + { url = "https://files.pythonhosted.org/packages/bf/19/64e52eced9a66cbb61dd40b110dc25297b1185f1027016876ae366fcaa1e/ddtrace-4.15.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4eb19b57f1cc6ab8e13e57b8257b91f6b0e5bede753320b30e3920a28d4b1a4f", size = 10833139, upload-time = "2026-10-08T09:44:53.716Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ae/0d964c6eb264ec0ab797071db48221acca6990b5837f1a6d804b88b818ec/ddtrace-4.15.6-cp310-cp310-win32.whl", hash = "sha256:740a87a6504617ec7584d5afb2a6169d780571ddc2b44270274aa8291a70a03c", size = 6756230, upload-time = "2026-10-08T09:44:55.961Z" }, + { url = "https://files.pythonhosted.org/packages/0a/2c/35a05709d630b3ad6f5c05dd90d34bf784fcde93e0722f9e1cec111ec5c4/ddtrace-4.15.6-cp310-cp310-win_amd64.whl", hash = "sha256:7aa9f30da1832111de778a17a26bc2bc564f4810d573ac2d8633609c3db48306", size = 7387627, upload-time = "2026-10-08T09:44:57.786Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b5/3ce5d64ed92d87388a83b52ed766866177997585e224d018922b0494a88d/ddtrace-4.15.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b3f6e66e3b303252edce525ca573e7092e741049556ae80a395eadf0685a036f", size = 7997045, upload-time = "2026-10-08T09:44:59.612Z" }, + { url = "https://files.pythonhosted.org/packages/f7/25/b7f15af30cf446fa95bf6835e515c135cc801ddc9ec18ecd665b7a82063a/ddtrace-4.15.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6bec8671c1e0bdadd7fad2513f449c07b9bf4f847bc652aef2720f0e2f74aeaa", size = 8381818, upload-time = "2026-10-08T09:45:01.325Z" }, + { url = "https://files.pythonhosted.org/packages/d9/95/1c597a597123eb99fccd0fc0ef64ee25ad46460e158dbf258136126134c3/ddtrace-4.15.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3017a4d4fe8375ae4555b80419d5259bdc444af485bcca6ee52dcd1a41890d35", size = 9541087, upload-time = "2026-10-08T09:45:03.274Z" }, + { url = "https://files.pythonhosted.org/packages/55/88/7e8c66d3db4739599a8092197940d211a006edca56b8fbbf9bbd34869a27/ddtrace-4.15.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:220fbb49481c6dd85866110477183b70882cb2578020c3bbbb4db2d89d0b41a8", size = 9793899, upload-time = "2026-10-08T09:45:05.384Z" }, + { url = "https://files.pythonhosted.org/packages/84/f8/08c6b74b9fd13df49cfdb95e6a1a86e06c434994db8f4a4a3395f0fee013/ddtrace-4.15.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d50b28793f381bb53e2d7cab5ffeb97c1aa0032f436a1fd5984cb3740d75b10", size = 10534489, upload-time = "2026-10-08T09:45:07.549Z" }, + { url = "https://files.pythonhosted.org/packages/44/6c/03cd4a5cc888f7c540fa9282d3c7c3a7675aae2c622c41668d1a765e0699/ddtrace-4.15.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b5a9edbd149d793b134bdaea28f40db5a199d42135f122c2be325519db55fb75", size = 10834145, upload-time = "2026-10-08T09:45:09.661Z" }, + { url = "https://files.pythonhosted.org/packages/25/6d/ab22cf125e6618597c41b55682dc76617cb6e071ca7381aac346840d6404/ddtrace-4.15.6-cp311-cp311-win32.whl", hash = "sha256:6b6a59fc3dfb081fd7708c9431b94dd17c5148625ec84bc425852cdfbf2f214d", size = 6752572, upload-time = "2026-10-08T09:45:11.785Z" }, + { url = "https://files.pythonhosted.org/packages/b6/38/ffc989894b1d7ea7c06fb249185807c574da5b4572adb9c974fcf02ea65d/ddtrace-4.15.6-cp311-cp311-win_amd64.whl", hash = "sha256:fef3cdf314eee555c5f6b831aeeb826a7eec147e3e1ab85591154a0ad96828f9", size = 7388927, upload-time = "2026-10-08T09:45:14.111Z" }, + { url = "https://files.pythonhosted.org/packages/ab/b8/1b8483fac4a4bcf1d7338d6323b2a3bde576553b111c5a288aee71224613/ddtrace-4.15.6-cp311-cp311-win_arm64.whl", hash = "sha256:4f163cc7fd9d76711c9df54273cbc2bb15933d90d1027fbf6ea3e938f042d23a", size = 7391090, upload-time = "2026-10-08T09:45:16.749Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3f/010c67d812aa0950bb44aeabec69c858e5bd7d31d9fe560ae431e2a08d95/ddtrace-4.15.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c4ca75ae638e105e5c28b7f7b3e1d3762f027ab14a315530e62c71d402b49dc1", size = 7993463, upload-time = "2026-10-08T09:45:18.638Z" }, + { url = "https://files.pythonhosted.org/packages/16/e3/c42f708e8506a75621678e5aac5219b03c8d95014b70e5755287cbe81d2a/ddtrace-4.15.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2f533ad60bb9855ab47a429eeb8b7e28667647c1a482899ac6bd5c00aa525407", size = 8387610, upload-time = "2026-10-08T09:45:20.667Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c5/ea86a9859f8e9b69a448c1ccbb70c51c7989c8c7bf68288356b9f53883a1/ddtrace-4.15.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e00a2943adf9cc886da3cf377b6c32001828565c877a9f0a7905e60eda527dad", size = 9514941, upload-time = "2026-10-08T09:45:22.929Z" }, + { url = "https://files.pythonhosted.org/packages/8d/af/b3bd0fdacb0d2b6baf8bd033ea00e97b06e74e3e504918ecf2f4d176ad6b/ddtrace-4.15.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dfb419eecf9ab3ad774e0fa612f08814530ebd2e1f0aa10128bb0f98eb7cea2e", size = 9772538, upload-time = "2026-10-08T09:45:25.295Z" }, + { url = "https://files.pythonhosted.org/packages/ce/05/f1cb0e9007a5c0d155948726bfefe97fc5fc9d7e886a951a4d51bc1c0236/ddtrace-4.15.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7f8bf477362e29742a6b282fbbfc9c6985c745ba302e4b6a8a53388911ceacae", size = 10511565, upload-time = "2026-10-08T09:45:27.732Z" }, + { url = "https://files.pythonhosted.org/packages/09/90/9f27058af312b213abfb946ab27b334b58b5482aada8b00ce2c41bee2ee6/ddtrace-4.15.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd6c6cd99d6edcffab804755f8246f3c1ecf727bdb241ecfe94f5de31fbb484b", size = 10824106, upload-time = "2026-10-08T09:45:30.202Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9a/f4402bdfb563d103ece19bd56a4622accd0113dfb9ac9b118050757823f9/ddtrace-4.15.6-cp312-cp312-win32.whl", hash = "sha256:903a24868a743e035ff0f7bdc7e397a90c00fac77b551765a9ceae5537c40a44", size = 6756938, upload-time = "2026-10-08T09:45:32.864Z" }, + { url = "https://files.pythonhosted.org/packages/55/fb/0b24d9fbcecfeb72ac2fb24db5b4a4684a0f0ecda73bb2b0c56c14282645/ddtrace-4.15.6-cp312-cp312-win_amd64.whl", hash = "sha256:57cd4ca85c5fef665fc2794c37369e39d6b9f51b1ff37ae8e68ac935360d1561", size = 7375054, upload-time = "2026-10-08T09:45:35.014Z" }, + { url = "https://files.pythonhosted.org/packages/b6/34/09763e279fb865196966cc895fb234cf12350f08fde8a284c796f13d6d34/ddtrace-4.15.6-cp312-cp312-win_arm64.whl", hash = "sha256:be11b8c8fce20b6e02d6069a53d43fc92aec9a50741e0f0dadebd913f2917ae5", size = 7376141, upload-time = "2026-10-08T09:45:37.46Z" }, + { url = "https://files.pythonhosted.org/packages/17/0c/f0b7dfb83917ea3f9c0d072cf8cf9454550ecf26372b19be49b8cf16bbcb/ddtrace-4.15.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8a118c85925d143e88e48e4583500e95a9b88d9c1fabc96df276c086eb515603", size = 7987311, upload-time = "2026-10-08T09:45:40.025Z" }, + { url = "https://files.pythonhosted.org/packages/f5/01/f39c527b10ceef714c16a99d59429ddd8b314c7d5301bff86fab6201dfae/ddtrace-4.15.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:1307ee87674c3d2d93f38bcd258b62b70a41081183e4bc94b03594c855751322", size = 8382130, upload-time = "2026-10-08T09:45:42.316Z" }, + { url = "https://files.pythonhosted.org/packages/e2/29/0c46ee3d6dcce28f8bd82c1dc658e420bed935fc0488a1ffac2853b42b0f/ddtrace-4.15.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fabb658b1ec068ab6046fb1ecaa6e6860bd40338235244c3c156c973c4957c00", size = 9507940, upload-time = "2026-10-08T09:45:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b0/77d18cb89ff7b2652d734ae00e4869ef703945a0393bc465e00878631099/ddtrace-4.15.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2185df0b730e4bb78dfe40cf61b7e242d4b81724cea4f496ca5a0ae0f5427309", size = 9764498, upload-time = "2026-10-08T09:45:47.528Z" }, + { url = "https://files.pythonhosted.org/packages/64/06/7435d3534b646ddafaf16fced76b0dec8d239492eef1e119d4c3b3fe1259/ddtrace-4.15.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7fc21e13b52290d33bcdec36644eb63eac03f49adb0b36c6d84dc4bb02b9b935", size = 10508643, upload-time = "2026-10-08T09:45:50.63Z" }, + { url = "https://files.pythonhosted.org/packages/34/d0/65db05d5b72fcb88378edd4bf44e88b2dc0c3e8e1c3070a4630e5ec2dfdd/ddtrace-4.15.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:008a5c93db9489861394190b9a9ecffc464a9a83882792c83c005e938fc69df0", size = 10817707, upload-time = "2026-10-08T09:45:53.417Z" }, + { url = "https://files.pythonhosted.org/packages/df/e0/b2ad40242ce8d17ba4432c4f96a7584d93852c442fcb8ac1285cbd10ae96/ddtrace-4.15.6-cp313-cp313-win32.whl", hash = "sha256:d5ca2d38c37194aee97801775193a034f7f27f2610d5075b6a550ee664b044dd", size = 6754044, upload-time = "2026-10-08T09:45:56.603Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e3/7a549bc05e6b604ef3c5ad8fef19a951178d8d93ee7088660d3743e53b7b/ddtrace-4.15.6-cp313-cp313-win_amd64.whl", hash = "sha256:fc35805f2c5b64bd7a29304a5c54932792a47b1436d2550354c051870cd706b4", size = 7370629, upload-time = "2026-10-08T09:45:58.955Z" }, + { url = "https://files.pythonhosted.org/packages/50/dc/a140b6640dd57f143e9137e8e4d04653ba9047c7ef65ff26ca03721c90ff/ddtrace-4.15.6-cp313-cp313-win_arm64.whl", hash = "sha256:03221f056c03728b01a4295ff4a3854db2dc33bcfab3131757f5eb4ca2a5a41b", size = 7378982, upload-time = "2026-10-08T09:46:01.698Z" }, + { url = "https://files.pythonhosted.org/packages/62/de/448c66ce790d6fcd1abbeb08e8e88ef1a5b7fc990f6ec48b818b4d9a8b29/ddtrace-4.15.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:70d5dfc73d970fa1b633e0aad3b9457cb94478958976e139b1c78461a3c48b33", size = 7993200, upload-time = "2026-10-08T09:46:04.284Z" }, + { url = "https://files.pythonhosted.org/packages/2f/87/50bbd2d4fe589aa13038c1fe07defd57e63c1bc83c22b92eac13676b3219/ddtrace-4.15.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:26ead4c2956924d0574b6681d063dee6c8741dca98ed3b747c353f6252eae0d7", size = 8388799, upload-time = "2026-10-08T09:46:07.071Z" }, + { url = "https://files.pythonhosted.org/packages/9e/13/d947c88b0fdd7cddb89174f5e9116aead05a935e1406e2921c820cb7bcab/ddtrace-4.15.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e62ec9c83e9e98122f4544592a6bf7d3338e9ea7ff2e34fe4750ade2b2c3e5", size = 9519810, upload-time = "2026-10-08T09:46:09.975Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/15711bfd54778a28cca9f7988bcdd03e4d680dfceed7f4db22f6b7a06e73/ddtrace-4.15.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3684507243feb45e7438b68989aae9a7df01b7996112d7fe74846e1d5aeb3726", size = 9771503, upload-time = "2026-10-08T09:46:13.009Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d4/d0e81dff83be9370b90c090828330c4b2f06d74644f331d8f6d9518bad44/ddtrace-4.15.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4882e25808cf6102015d7421d6214aa94a788ce3242fec9a1a726a41dcfd79ac", size = 10521211, upload-time = "2026-10-08T09:46:15.867Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c1/e33b44ebfa4dcb80f66fb2cc99c61d9c7706edd48e5ccb5ad04bcea0d5ad/ddtrace-4.15.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5336630ed8c829333dd4195d817b3f3f8211ddf6074e1c731e884771094fc41c", size = 10826453, upload-time = "2026-10-08T09:46:19.084Z" }, + { url = "https://files.pythonhosted.org/packages/e5/32/2125cf7f03e9e8b68b6a9072e393a3be4af91505ada32bc65811d6965881/ddtrace-4.15.6-cp314-cp314-win32.whl", hash = "sha256:a970bc70c81361c9a27452bfd3335182d722315466109208290a1c3a4c221748", size = 6867770, upload-time = "2026-10-08T09:46:22.66Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a0/c18b1db1cf4843a3254aed03b61c1c9cbd5964bd52c18922f633c7ed2304/ddtrace-4.15.6-cp314-cp314-win_amd64.whl", hash = "sha256:e7e8b948d13737e61a8e0dc1fe082d38431a3d4fb285d23236f1c8f733ee5c67", size = 7538347, upload-time = "2026-10-08T09:46:25.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/77/6181506d7f7693aceccdc13f5a2dc6b18662ed3eedebc02043f00d502e5e/ddtrace-4.15.6-cp314-cp314-win_arm64.whl", hash = "sha256:11662a7a4e1f51c388450cd1092b57738bb32ef98401c6032754a7e220a5f6d0", size = 7574014, upload-time = "2026-10-08T09:46:28.185Z" }, +] + +[[package]] +name = "ddtrace-internal" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/51/6dc02752dca5920d6627fb92b7e0611f594a823bbd57ed31167571d819bc/ddtrace_internal-0.1.0.tar.gz", hash = "sha256:7390721276ab5f174cb103088be7d0de22d452f2817c949e8bde1182ff5b961e", size = 2504, upload-time = "2026-07-15T18:35:10.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ad/3cae600fdfc41f54653c14339ac1f125381a6fc2a823d5e685b627ff152e/ddtrace_internal-0.1.0-py3-none-any.whl", hash = "sha256:8eeac4a812e003cebafc8be35ef05103ae2b132893dfbcee00a67f4af81404a2", size = 2495, upload-time = "2026-07-15T18:35:09.295Z" }, ] [[package]] @@ -4991,7 +5001,7 @@ source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.100.post1" +version = "0.4.100.post2" source = { editable = "litellm-proxy-extras" } [[package]] @@ -5657,75 +5667,77 @@ wheels = [ [[package]] name = "msgpack" -version = "1.2.1" +version = "1.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/e7/bb605a7bab2d8425a64b3fa762b39dc1bf1c7e3f11ba6fb5413d6db0ff8c/msgpack-1.2.3.tar.gz", hash = "sha256:32edb81a2b5eb7cd7c9d941b2bfbbb082fd2cd09e0e725930316af6b708db186", size = 196517, upload-time = "2026-09-29T02:33:52.276Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/16/f70100614b69feb3ade7285f08c9c52d6cda0a5c03f3f5e2facd63acb211/msgpack-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c7b398c56ff125feae96c2737abfec5595f1fa0aa186df60c56040b8accb95c", size = 82926, upload-time = "2026-06-18T16:12:31.531Z" }, - { url = "https://files.pythonhosted.org/packages/e4/3c/08ecd5cdfe4e2de43aec79062028ad0f7b2d9b1fea5430068c198ba570da/msgpack-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1548006a91aa93c5da81f3bdcebc1a0d10cea2d25969754fbe848da622b2b895", size = 82730, upload-time = "2026-06-18T16:12:32.894Z" }, - { url = "https://files.pythonhosted.org/packages/19/9f/a70c9cb1a04ecc134005149367dcfe35d167284e8f65035a1e4156ad17b5/msgpack-1.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dabedcd0f23559f3596428c6589c1cd8c6eaed3a0d720795b07b0225d769203", size = 400729, upload-time = "2026-06-18T16:12:34.052Z" }, - { url = "https://files.pythonhosted.org/packages/fa/7f/5ce020168cf0439041526e95aa068c722c016aee21624e331aeabeee2e8e/msgpack-1.2.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83efa1c898e0fc5380fc0cabbf75164c52e3b5cbb45973710d75821928380c73", size = 407625, upload-time = "2026-06-18T16:12:35.239Z" }, - { url = "https://files.pythonhosted.org/packages/79/70/fb7668ce0386819303047057aef6fc1da73b584291d9cff82b821744e2ef/msgpack-1.2.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01e2dd6c9b19d333a00282330cc8a73d38d8dabc306dc5b42cd668c3ac82e833", size = 377891, upload-time = "2026-06-18T16:12:36.684Z" }, - { url = "https://files.pythonhosted.org/packages/3d/dc/9ebe654a73c3aed2e40aa6b52e3c2a02b5f53ef0085fa235a45d5b367f87/msgpack-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:350cb813d0af6e65d2f7ef0d729f7ff5be5a8bce03665892f43e5883d4ecc1b8", size = 391987, upload-time = "2026-06-18T16:12:37.839Z" }, - { url = "https://files.pythonhosted.org/packages/42/eb/b67cf64218a2fa25e1c671fe1d3dbb06cbeb973e71bc4b822da079862d0b/msgpack-1.2.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ee1d9ed27d0497b848923746cf762ed2e7db24f4be7eec8e5cbe8c766aa707b7", size = 374603, upload-time = "2026-06-18T16:12:39.221Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2e/9ee200cde32fd1a0101b4006202fde554c1860adfb9bf7bff31ea4c08df8/msgpack-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:633727297ed063441fd1cda2288865487f33ad14eeb8831afb5f0c396a62cfce", size = 405121, upload-time = "2026-06-18T16:12:40.524Z" }, - { url = "https://files.pythonhosted.org/packages/43/b6/f10117be7ca7a51e8feed699a907b8e663a8cd66e115ae6b4fb30cc7945c/msgpack-1.2.1-cp310-cp310-win32.whl", hash = "sha256:298872ecf9e61950f1c6af4ca969b859ee91783bb920ef6e6172697d0c8aad74", size = 64088, upload-time = "2026-06-18T16:12:41.762Z" }, - { url = "https://files.pythonhosted.org/packages/ba/93/89976c696fb0224662239d952c47b4d1661b34d79a332ef5584facaa8579/msgpack-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:2ff164c1b0bcb740b073b99e945234d0212852fa378e44a208c425379140dbeb", size = 70113, upload-time = "2026-06-18T16:12:42.78Z" }, - { url = "https://files.pythonhosted.org/packages/f4/6b/e9b1cdc042c4458801d2545ed782a95f3d6ba8e270cce8745b8603c7f748/msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22", size = 82812, upload-time = "2026-06-18T16:12:45.022Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3a/dd518a1bf78ed1e9ad8afe57307c079a00eafe4b3068932a27ca1ea56b4f/msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5", size = 82739, upload-time = "2026-06-18T16:12:46.025Z" }, - { url = "https://files.pythonhosted.org/packages/70/e0/7ba9e1542bf0771a27b8b37c1316e3f95ae9d748fd765284655c476ad4ef/msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06", size = 414233, upload-time = "2026-06-18T16:12:47.029Z" }, - { url = "https://files.pythonhosted.org/packages/03/8d/671d81534ea0e2b0e8a121be100020da09eb78861fe3aa8f3ef7dcd3bed1/msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4", size = 423843, upload-time = "2026-06-18T16:12:48.19Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b6/e5c737515ed1f166664b87601b532f58cbb73d8aa6a90b99f7c2c5037e8e/msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8", size = 390772, upload-time = "2026-06-18T16:12:49.624Z" }, - { url = "https://files.pythonhosted.org/packages/a8/46/62ed8c2e87d7021eab19921594d961ef3aa3794eec76c716dc30f3bfd433/msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b", size = 409559, upload-time = "2026-06-18T16:12:50.936Z" }, - { url = "https://files.pythonhosted.org/packages/70/ff/59aa3887b860bbf43532835e192b1c388a17590d6068ae4f8b2bc74c906e/msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e", size = 387838, upload-time = "2026-06-18T16:12:52.161Z" }, - { url = "https://files.pythonhosted.org/packages/09/11/f8563e471093420cf6478cb3271a0175d8402b82d879783d4035d2d03360/msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f", size = 421732, upload-time = "2026-06-18T16:12:53.556Z" }, - { url = "https://files.pythonhosted.org/packages/57/cf/e673683c4c6c90c1022b24c65af4b03eda72b182a1176ef6449069d66acc/msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d", size = 64091, upload-time = "2026-06-18T16:12:54.821Z" }, - { url = "https://files.pythonhosted.org/packages/3f/07/ca212739d179f9083bff2c7c08c24101c3555a334fadc2b876b18768a3ae/msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8", size = 70462, upload-time = "2026-06-18T16:12:55.898Z" }, - { url = "https://files.pythonhosted.org/packages/6d/be/6798347b425e26f35db82e69dd83c09716c856a3714e7bffc4c0860fd830/msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66", size = 65059, upload-time = "2026-06-18T16:12:57.053Z" }, - { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, - { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, - { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, - { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, - { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, - { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, - { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, - { url = "https://files.pythonhosted.org/packages/b0/ac/dcddcab6f6c20ecb387ca5e980371cdb3f87ff69aeca388be97eebc4c074/msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064", size = 83151, upload-time = "2026-06-18T16:13:12.173Z" }, - { url = "https://files.pythonhosted.org/packages/64/71/fbcfa83a1d6a9c6091942d1cfd070962244664b87427a9a49a6897b1b219/msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056", size = 82351, upload-time = "2026-06-18T16:13:13.194Z" }, - { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" }, - { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" }, - { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" }, - { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" }, - { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" }, - { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" }, - { url = "https://files.pythonhosted.org/packages/ef/c5/133f4512a56e983a93445c836c9d94d88f3bc2e0980ff4b9e577bd8416ce/msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707", size = 64471, upload-time = "2026-06-18T16:13:22.293Z" }, - { url = "https://files.pythonhosted.org/packages/e2/98/577e10b055096a7dd40732358cabaf7180a20c79ed1dcdbb618e4b9deac7/msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9", size = 71274, upload-time = "2026-06-18T16:13:23.455Z" }, - { url = "https://files.pythonhosted.org/packages/ba/ee/0c0048e7cfbef23c6a94791b8959ab28155232e7956de8a305b5ff588f05/msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a", size = 64795, upload-time = "2026-06-18T16:13:24.687Z" }, - { url = "https://files.pythonhosted.org/packages/77/58/cce442852c6b9e1639c7c8ac8fd9143121cb32dab0f308df4d1426a8eb9c/msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d", size = 83610, upload-time = "2026-06-18T16:13:25.724Z" }, - { url = "https://files.pythonhosted.org/packages/60/5c/15b4c7a0182f75ffa90751958ba36a9c01cafee367d49a3edc10ed140b01/msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7", size = 83138, upload-time = "2026-06-18T16:13:26.781Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" }, - { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" }, - { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" }, - { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" }, - { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" }, - { url = "https://files.pythonhosted.org/packages/82/90/8b630fef07d8c5ab457b71ff2c217910c83d333c7a68472c186e87cc504a/msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb", size = 65942, upload-time = "2026-06-18T16:13:36.056Z" }, - { url = "https://files.pythonhosted.org/packages/16/f1/467b81e98b24dd3885d7b1857728797b4ffc76a7a7483af4fb321a07de3c/msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b", size = 72627, upload-time = "2026-06-18T16:13:37.079Z" }, - { url = "https://files.pythonhosted.org/packages/a7/1d/5d8c4c89985feb6acefb82a09e501c60392261856d2408d20bfe4f0360b1/msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7", size = 66908, upload-time = "2026-06-18T16:13:38.23Z" }, - { url = "https://files.pythonhosted.org/packages/1b/02/ad2afb678b4de94496cd432b581759b756a92c1192d8c767edd6b132efdc/msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273", size = 86000, upload-time = "2026-06-18T16:13:39.44Z" }, - { url = "https://files.pythonhosted.org/packages/54/74/0b797484013128837f3b1cbb6cea019277c4de4e377dc512b4d9a0f92940/msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1", size = 86544, upload-time = "2026-06-18T16:13:40.447Z" }, - { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" }, - { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" }, - { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" }, - { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" }, - { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/aab6c946570496b78e67804721f3d5e2d62a93081b9b37df77764ef56347/msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1", size = 70914, upload-time = "2026-06-18T16:13:49.385Z" }, - { url = "https://files.pythonhosted.org/packages/13/0a/e608956488a2af014cfe6e3d665e090b8ee42aa14b07f8f95b8880d66b09/msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2", size = 77999, upload-time = "2026-06-18T16:13:50.467Z" }, - { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" }, + { url = "https://files.pythonhosted.org/packages/6d/aa/5b6b09f835791045282dc5d08431db599a5f4743a69fe2f6670045a2cd85/msgpack-1.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec0030361cc861ac699b2ef1c695b741fa145c88f8667fa3d7e3f73deeb648a3", size = 90927, upload-time = "2026-09-29T02:31:28.286Z" }, + { url = "https://files.pythonhosted.org/packages/c9/91/7b288e9133bd1ba92ca0ca4e7f2a4cfc53cf467d99d8d2f57b9939908fac/msgpack-1.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5c1efdd9181cb1b719ee46865f368a927f1c0c65d577798340b1194545b7515a", size = 89798, upload-time = "2026-09-29T02:31:30.028Z" }, + { url = "https://files.pythonhosted.org/packages/71/9b/5c3dbc450d14645dcec987970692d6ab24008cc33d2155474b1d818486f9/msgpack-1.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c309a7abae1d14ba29a8bd0ddbd704a5e469d8e9bd9c3dee0e4ff53d7ae01d56", size = 450687, upload-time = "2026-09-29T02:31:32.407Z" }, + { url = "https://files.pythonhosted.org/packages/2b/21/ea60a8fd0d9e0897fce823e9fd9bf6742567784b35c7eee8f4a18a56eb19/msgpack-1.2.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bf390259cb25a6a1cd197c65810999b811f64cd38683251538bcc5a1e41f7d3", size = 459808, upload-time = "2026-09-29T02:31:34.282Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f7/42140e6afdac8e94bfedae4cfb67ee004b6ad5c4cadd024df42f759bf3b5/msgpack-1.2.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39b6986c19e1f2dfa549d185dba6ccf1de2e4c0ba10d8cfc0048935b1c5f9109", size = 423845, upload-time = "2026-09-29T02:31:35.713Z" }, + { url = "https://files.pythonhosted.org/packages/19/7b/cd54f27b59dfbdc438a12361fbb6798b66d377a978f946bc9512598290e9/msgpack-1.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fcc6800daac4922960f6eeb7a0dda3dd4105e0bf7bce0e83ebc465a78cb7bdba", size = 445608, upload-time = "2026-09-29T02:31:37.65Z" }, + { url = "https://files.pythonhosted.org/packages/57/38/52bc0dc44cc9f7c2339b632f93d02f8badc78cfb0bb070f2a50a51945e53/msgpack-1.2.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:968583e956d0427878050b371308c5f8647088732ef3e66a117dbe1192ec91e0", size = 421721, upload-time = "2026-09-29T02:31:39.151Z" }, + { url = "https://files.pythonhosted.org/packages/89/e6/451c9a42274fb2be82d8ba8b76a5219c613e20f8de1da521d10cb758a9ef/msgpack-1.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1d6bcec3dbbdb89ca385d3a73e63ceae7b841fa0d7ca7c676f1a7bfe7fb2cdb8", size = 460430, upload-time = "2026-09-29T02:31:40.843Z" }, + { url = "https://files.pythonhosted.org/packages/57/bb/663e3100327b58caaa5fb66379e557a2717dac08bb586f22f885756bee47/msgpack-1.2.3-cp310-cp310-win32.whl", hash = "sha256:a6b63917d60d6df451f328bd6afba8565e33c4afe1f62ec4ad758b78731c827b", size = 67987, upload-time = "2026-09-29T02:31:42.157Z" }, + { url = "https://files.pythonhosted.org/packages/28/7a/a00d5d7abc5601099260e0d0af8fadc54fbfac2191315aa56eaee3641d9d/msgpack-1.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:4c0780095871ecc49a58b2ff6b1b43b25214704da67646557ca287a3f49fb2dd", size = 75572, upload-time = "2026-09-29T02:31:43.544Z" }, + { url = "https://files.pythonhosted.org/packages/2a/95/b9c651ccb9d720b2e2c8d537954dff528ab869a03bf89598145716db823c/msgpack-1.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ec90a9ae3e1169fa1171147340f0e97d941aa19fcd3b34e8339a55933ed042af", size = 90404, upload-time = "2026-09-29T02:31:44.826Z" }, + { url = "https://files.pythonhosted.org/packages/50/cd/fc9e2e367e80f1493e2ec5f610dda558b344eeede296f88976db133e8f2c/msgpack-1.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d7e9cbb0998bbfd363fd9a09c330520d5e9cb323c05b5a1a05865d23ccf2226", size = 89683, upload-time = "2026-09-29T02:31:46.413Z" }, + { url = "https://files.pythonhosted.org/packages/19/9e/1028485c6886c1c117f777cc9b053e541eff0fedb3292dfb1da95040edb5/msgpack-1.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6707d2fa2aa1bb5424ea0b05f44ffc989b15ab41a73ff5855bff4944fec7c8ac", size = 465347, upload-time = "2026-09-29T02:31:47.934Z" }, + { url = "https://files.pythonhosted.org/packages/aa/83/800570e6a22376eb8d599920f70aead4779a63611696f567477c4e85a70f/msgpack-1.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:382b219de3d436de3baba0f4b0c6d4336e8f5858d0eb047918b13b69a71c6c55", size = 477820, upload-time = "2026-09-29T02:31:49.479Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ff/817e4a2052f848d3fb67726908d6e4e7c19f68ee7c19553a82ce7b0ed415/msgpack-1.2.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:186e6c602b8a9968b8e864c67d622a69279f7d1e55ae25f40e3bff7e815b2b62", size = 436656, upload-time = "2026-09-29T02:31:51.18Z" }, + { url = "https://files.pythonhosted.org/packages/3d/42/040cc55dde6a7d92057baac8d1fc9cfb9f4fd4162900e2ec16dc33917a7d/msgpack-1.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9276ba88891338f2617044429dfd080ae008c9868a25f6f1a7d004a35dc9ac0a", size = 460939, upload-time = "2026-09-29T02:31:53.026Z" }, + { url = "https://files.pythonhosted.org/packages/09/93/4dc007bdef930eed247346773bc0189b710078961d3218d5ee7ba59f322c/msgpack-1.2.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c942c21a93f36b3a69e828c8945bb72c94dc2ffe488a2086950c812f3edf046c", size = 433608, upload-time = "2026-09-29T02:31:54.981Z" }, + { url = "https://files.pythonhosted.org/packages/c0/97/a1b944046f283ec89445cb2a982c42233b5b07cc630f9be739f4f1d469a3/msgpack-1.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18a6ed513023001b28dcd3ba54966f6bb90a38274ba8d2640464bcab3a1b81d4", size = 477373, upload-time = "2026-09-29T02:31:56.713Z" }, + { url = "https://files.pythonhosted.org/packages/59/79/ab411d0d172743732ab2503f4c32a22dd1a7d1436a6feecbb160e4b6376a/msgpack-1.2.3-cp311-cp311-win32.whl", hash = "sha256:d0238cd05dec9ffbe0de1071df685ba63e30a36ac155285b1a094e727c38cbe9", size = 67514, upload-time = "2026-09-29T02:31:58.267Z" }, + { url = "https://files.pythonhosted.org/packages/63/8d/6f0cb2b84e484e96278455c26870196d025bb0cec312b226a663f1fa9000/msgpack-1.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:30e1522e4173230dca4d9ad896f038f73c0da6c1edd42f4dbad88ac583cf5d46", size = 75850, upload-time = "2026-09-29T02:31:59.449Z" }, + { url = "https://files.pythonhosted.org/packages/aa/25/f99e13a2c1d3f5a1dcaa5aab27f474e8c4358188bbc68ad79fecb0d1aefe/msgpack-1.2.3-cp311-cp311-win_arm64.whl", hash = "sha256:8ca67f77938ea6a3663aa9bd22b3e031f6da84d665be850abab910ee90728dfd", size = 72338, upload-time = "2026-09-29T02:32:00.885Z" }, + { url = "https://files.pythonhosted.org/packages/af/12/4d7c6d6203416d9fbf0f59ebaa805e70fb929b93a41b611bc821ec5964a0/msgpack-1.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:89c930aece4e972b208ba589c8410b4167b05e411a5ea2cb25fd96f8bc47ee43", size = 91577, upload-time = "2026-09-29T02:32:02.141Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c7/8576ad39f4ca42ddad26f68eb8621d2d0a60501193d480f504bd9d7f36c4/msgpack-1.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:905a189853d6bdb204c7ae5f4ab77fb857448abfff574d3d93c62e2815b24b4f", size = 90027, upload-time = "2026-09-29T02:32:03.508Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3a/aa9c580aea1314529a0f3562461479780b0d254b064f0880956bfbcc74a8/msgpack-1.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3d7b3d0018746b5997dd6b14a1870b07cc4c327d9101145d94a1fc264a51a06", size = 460343, upload-time = "2026-09-29T02:32:04.906Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cf/9c2e4d6c179529d5bf4a64cff76fa581486569e9fbdd35bd98f51cb624bf/msgpack-1.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede33b2892ceb976283e009ad12fa1834cfdf1f9c43ee9c97849fc588d00a618", size = 472998, upload-time = "2026-09-29T02:32:06.69Z" }, + { url = "https://files.pythonhosted.org/packages/7b/41/915c81fe6df2d3cbdb0dece4f1a5cd313e1cd2abd9f501d0f50c0582517e/msgpack-1.2.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:666ef5601ab0e6e345e47febc96aa81143cc932201543480cbb9499164f05ffb", size = 423216, upload-time = "2026-09-29T02:32:08.739Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e7/7dda8b1039abfd9bba4c5068172c67135c9e33089f503512db9226f23c24/msgpack-1.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87cf2ef05ff2f2493ba29fcdaef27e960ca64dacfd13460ae29e6f92e0ed05bb", size = 451218, upload-time = "2026-09-29T02:32:10.517Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/ce995c1ed4a0522b7f2d034bc2034fd63005f240b945961b70fb56fbaf3d/msgpack-1.2.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b774ff994d844e541439ac5d2d49a14def4104830c3465e9394c153f86200ffb", size = 422453, upload-time = "2026-09-29T02:32:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/ce191fb87e2650d0166b34c437e499ee4a7f9db9c1eb164f41725eb6160e/msgpack-1.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaf7e82249837e3aa97297b34a0bb9ff562027381631e057cea6e1367f10b438", size = 469003, upload-time = "2026-09-29T02:32:13.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/35/539123407fe200fb16609c835675496fbeb6017ace9fc93909f0613223ae/msgpack-1.2.3-cp312-cp312-win32.whl", hash = "sha256:7c047250096f9fc19dba26e3d1639b5e7a84114003605c94def667149a70ced1", size = 68303, upload-time = "2026-09-29T02:32:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/6f/4c/331b45f9b86fbda6b9e103244d189068e51f726d8c40021ed66e1f2c415e/msgpack-1.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:3ec409b0d6aa8e9eec6eaf881b893caa215dbe68c5319ca96e8a271d81bb111d", size = 76744, upload-time = "2026-09-29T02:32:16.344Z" }, + { url = "https://files.pythonhosted.org/packages/13/9f/fb572dc42b9fac06c7ea848aaee6e140d84469743bd1402bc07089fc4566/msgpack-1.2.3-cp312-cp312-win_arm64.whl", hash = "sha256:59612b4ed48a04cf024584218e813562f3b30a3bafa5f55abe300b15da314751", size = 71580, upload-time = "2026-09-29T02:32:17.617Z" }, + { url = "https://files.pythonhosted.org/packages/1f/8b/3824d65e912e925d09ce30d9130fa9970d6d2855d7888b13639a6604967f/msgpack-1.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:21bfa4d2aa0b04c1806ef778a1199e9e53ea2441bcbf284420a32083896320b8", size = 91728, upload-time = "2026-09-29T02:32:18.949Z" }, + { url = "https://files.pythonhosted.org/packages/05/e6/df7f2c9ebb94760113debbcea2bd3afe5fdab88a4f7bec1b618755517460/msgpack-1.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:db84203b13aecc222f465061397fdd5b53b7ae73d2c95ffc1c8dc5be0153a709", size = 89955, upload-time = "2026-09-29T02:32:20.224Z" }, + { url = "https://files.pythonhosted.org/packages/08/6a/e5fc57136e8bacccb2b39627dea2cd546540a06181e22fe6db90e15b3ae4/msgpack-1.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e0d7950ca3c1bbae291d0552dd3bb2792fc680629c4c0d44e47e5bab969f3ca", size = 454930, upload-time = "2026-09-29T02:32:21.771Z" }, + { url = "https://files.pythonhosted.org/packages/b0/30/c394d37898db9212d1693456cdf363c7e1a097d0b63e10664007f3df3ec1/msgpack-1.2.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:07c9733089d1b176c3dd2f7fa268452f9d5d784d076473499d754a58e8d1fbbb", size = 466866, upload-time = "2026-09-29T02:32:23.742Z" }, + { url = "https://files.pythonhosted.org/packages/4a/c8/1e4ddf6f6b829b3ee6c530c79dfae89cb609d2b0eedb5e0ae716851c52d1/msgpack-1.2.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f24a43b3560e20f825b807fe1e874bd73d53abaf8bbdcf258a6eb152cddbc1f5", size = 418715, upload-time = "2026-09-29T02:32:25.262Z" }, + { url = "https://files.pythonhosted.org/packages/11/a5/f460ba6d7a12d4301002f3efbb8f841e8bdc9c5fc98d771689677a352885/msgpack-1.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6576f348ed6cc4f31db6fd915a8e94245f042f50eae08d48732425e70638ea37", size = 446489, upload-time = "2026-09-29T02:32:26.988Z" }, + { url = "https://files.pythonhosted.org/packages/49/23/adface88db909bed321c85dd673655152d4a514c67e1f0800eb51c777d07/msgpack-1.2.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cd5a9f9f86a52c24713679aa2631956835f3842512964ff93f736ff76f1f530d", size = 416998, upload-time = "2026-09-29T02:32:28.606Z" }, + { url = "https://files.pythonhosted.org/packages/36/00/5bb3a239ccfc3763c4d0fa49b13b1b7010b00182c499ab3c1fecfe6294bc/msgpack-1.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f9ddd28d3e9bbc602a9dced1591882c7fb9ab776eef8837da2c326fde19e2853", size = 463288, upload-time = "2026-09-29T02:32:30.375Z" }, + { url = "https://files.pythonhosted.org/packages/29/8c/456df77f00d701df9d6980ffb80291bce6e4e2e112e25a4dfae216f0715a/msgpack-1.2.3-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:62cc1a4ef0e553bac32c8342e1f04834aca7de276b92744eb7307db77759b890", size = 53347, upload-time = "2026-09-29T02:32:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/9d/22/ce780be666f89b77cdb855daa9ec62e87bb7f69e9f403e4a5d83a2b2208f/msgpack-1.2.3-cp313-cp313-win32.whl", hash = "sha256:d2f9c4f85e47a44d26d5baf3b041eef23436e224d44eed273f01bd8a12048d9f", size = 68258, upload-time = "2026-09-29T02:32:33.163Z" }, + { url = "https://files.pythonhosted.org/packages/51/06/c3def9bc4db283103c5901b302ee2a4305cb1e69729244f94d9bd8f8e8e7/msgpack-1.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:bb89b5dc30469c84bbf8684826eb851d82412ca95690e111b9ac5e8fb343961a", size = 76569, upload-time = "2026-09-29T02:32:34.412Z" }, + { url = "https://files.pythonhosted.org/packages/12/9f/cef344073858b80adb92d6ea342e20b0eae7a8f6fe70281b69cf03707270/msgpack-1.2.3-cp313-cp313-win_arm64.whl", hash = "sha256:471e12a6a42498a31490c206e0069e343b6a7c35db540be73a879eb06f5be047", size = 71530, upload-time = "2026-09-29T02:32:35.892Z" }, + { url = "https://files.pythonhosted.org/packages/3f/8e/f777f74e38731c428857933c8011596f2d2f3160c821152f23b6ffba862f/msgpack-1.2.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3a31905206722103a84c1f72633fe30692cff6732c9d262e09a27dbc468797c8", size = 92042, upload-time = "2026-09-29T02:32:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/a0/71/551608543ee5d590f7e8d522267665d6d9946866ad2a2a70a770f7c70793/msgpack-1.2.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3372475211a9ce1a23acefe512cb3e121d18c95dc74ed56cb1819ef40836ebf4", size = 90578, upload-time = "2026-09-29T02:32:38.883Z" }, + { url = "https://files.pythonhosted.org/packages/ea/11/6d78ce5a9a58bf9ba7b1b6a8f649173b030e6770c8019cf330b91825ee5d/msgpack-1.2.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9324c54995641c3d1f92a9d55093c8cde0ffa2fbc87a467a688ef60428393220", size = 454352, upload-time = "2026-09-29T02:32:40.34Z" }, + { url = "https://files.pythonhosted.org/packages/3d/08/feb9a196269ba7809f44f9117d9e4a601c41c313f6144fd0c337293a5488/msgpack-1.2.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8ef3a66e4b52d2d7fdd90df2984670124b2ff7546d76bb25dcf68ef47f7df58", size = 462562, upload-time = "2026-09-29T02:32:42.176Z" }, + { url = "https://files.pythonhosted.org/packages/f5/77/3a674f366def24140b103d1ffd4fd27b3d912a13e47da67422afa16bebb3/msgpack-1.2.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:902f3490db0e07a7d40b48536a85c9b28fbf1397e7e1658a45a55f958e303620", size = 418134, upload-time = "2026-09-29T02:32:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/48/82/944e71f280577490d99a3951cbce21aa4cbe04e7ab42cb373fd668af883c/msgpack-1.2.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e51eca14fbb65c4e0a5a9657346962bd3dca78c08e04e3d4dee70ef48687d30", size = 445937, upload-time = "2026-09-29T02:32:45.739Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ec/feddd629c4a3edf1395313680450c525086cceab56dec0d4de9da9ccb618/msgpack-1.2.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f42f146752eedb6765f07dcc04d72dab0a25779ec8d4a88c0085263ce114f22c", size = 416450, upload-time = "2026-09-29T02:32:47.558Z" }, + { url = "https://files.pythonhosted.org/packages/e4/59/263a10f8c4613ba0713f48cbda7695ac8dd6d6fab2fcbc9168f03f23a94d/msgpack-1.2.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0ed5823c4efc20fe87d3530665f40ec18a002be003114814c21235cc8d256207", size = 459546, upload-time = "2026-09-29T02:32:49.145Z" }, + { url = "https://files.pythonhosted.org/packages/1e/21/addcfa1e583cfc8a22fbdc57526621b5decd7ad676ae12e9150b7be1be5d/msgpack-1.2.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:2487453ca1b6104442c6442f9a1a8fee1fe8f428a70d99d4cba799108b304150", size = 53462, upload-time = "2026-09-29T02:32:50.708Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2c/3cb5c8524a1335ee27ca952c7ab78d375a16fea8e18ae3767ba0c880416c/msgpack-1.2.3-cp314-cp314-win32.whl", hash = "sha256:6df430419f2338cb71e4a34d6e64f83c88ccd321f91f40ba4513400b36d864ec", size = 70294, upload-time = "2026-09-29T02:32:52.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/f9/9172ff3cdb85d160ad06df5e2708a5fce7682982a5eee8d31869b9f69d2e/msgpack-1.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:84a6616d396ec1bc18a1e83e67c96a393ec35dfe5e17434a5be7b9aa0fe988ab", size = 77778, upload-time = "2026-09-29T02:32:53.429Z" }, + { url = "https://files.pythonhosted.org/packages/04/e8/b4c23178bcf605ae17cec48a75530dd69d49b0a5a6f5f4df5c47d59f746e/msgpack-1.2.3-cp314-cp314-win_arm64.whl", hash = "sha256:7a003b02c6ee2eea6dfe0bb08818631e3597e69f0131f2a8250488a1cc553290", size = 73794, upload-time = "2026-09-29T02:32:54.763Z" }, + { url = "https://files.pythonhosted.org/packages/66/b1/92704be352c4f428b7e0a0e0fb210cb1aa2b1c42c102b8dc22d34b82fac0/msgpack-1.2.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ccea05b5542f6d283fef3f0a8e93a7f0be90af0ddeeef84c25c0216ba76dcae1", size = 93721, upload-time = "2026-09-29T02:32:56.342Z" }, + { url = "https://files.pythonhosted.org/packages/49/78/9c91f1e86cadcbc100b3780fd429c3715648704032a612e77a00646ebe79/msgpack-1.2.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b1631e12fe572e181cd77e831f69335d6cd5278eac22e3db3f33cf264ac2ac18", size = 94256, upload-time = "2026-09-29T02:32:58.056Z" }, + { url = "https://files.pythonhosted.org/packages/91/4d/270f9725921ae88a29d37a774a77ac24f0ef1411fc960a63f5a4665e81b4/msgpack-1.2.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e54394b7dbe2e12ab032d9d21feef7bb61a90a150a2623633ba3781ba69dcb1f", size = 471673, upload-time = "2026-09-29T02:32:59.886Z" }, + { url = "https://files.pythonhosted.org/packages/48/b8/eaa8d930f72dc1d1dd79511dc2ccf965922b059f2f0ed3b30aebac8c4b11/msgpack-1.2.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63bb7448a1e9111319ae2430c09a5596140c160422830d6271bc75730ff2ff9a", size = 466257, upload-time = "2026-09-29T02:33:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5a/97adc805037bc7e24c4e2f711bbcd3b28be8ec9aea3e778f18208cfbdb46/msgpack-1.2.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:382bc88fe90f29f5ac8a0b65c7046ff255356f2f2f3186c30e370215736fa1dc", size = 418484, upload-time = "2026-09-29T02:33:03.402Z" }, + { url = "https://files.pythonhosted.org/packages/0d/7e/1c53302606fe436ab48ba539ebafafe4a6a9efe12c4f04dc7eb36912d93e/msgpack-1.2.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c77e27790ad72989db783d5303825fba0b71550f00a490efba35cde7dc4b719f", size = 454064, upload-time = "2026-09-29T02:33:04.977Z" }, + { url = "https://files.pythonhosted.org/packages/00/2d/9ee0170f638907b396c15c6cd26b3e54f869159efc6206683acfd8f696e1/msgpack-1.2.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:700bc0fc9e968a292b9137ee70e7a012f7e115bf0107ce45e3a88202788dfc1e", size = 417901, upload-time = "2026-09-29T02:33:06.489Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d2/905c84490a75cd15a27065407cd085d201f7d392e1e0411f49f03fd31ade/msgpack-1.2.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bd5f91ea75c45cafcc5433ba8fae59b708b736ec178d2441c40c499e9e079db", size = 459896, upload-time = "2026-09-29T02:33:08.361Z" }, + { url = "https://files.pythonhosted.org/packages/37/cd/4ce5809b9ab3b114d7cca64863e436820fa1614b49d55ccb93d49824ac2d/msgpack-1.2.3-cp314-cp314t-win32.whl", hash = "sha256:7995a7c6a62a1d6e7df211b4a16de513bd99fd053525050a319f80f44fb8015e", size = 75983, upload-time = "2026-09-29T02:33:10.023Z" }, + { url = "https://files.pythonhosted.org/packages/8a/31/853bb580744c24be0dbd8b090c3e6987dce466a1fc840fe50c0ac2ef9044/msgpack-1.2.3-cp314-cp314t-win_amd64.whl", hash = "sha256:bfe7d5b62cbe7aa664f0b3e2c49077f10fcdd06183d3014f8271ff3c5edbfbf9", size = 83757, upload-time = "2026-09-29T02:33:11.441Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/9f1b2ee484414eef9e21ee2b2b23b482bb71433ab9bac1da03cbda15ebf5/msgpack-1.2.3-cp314-cp314t-win_arm64.whl", hash = "sha256:1f585407f740a9eac04a3bb82c61d68a0ea78f90e29e670bfb086b9ce3a518dd", size = 78128, upload-time = "2026-09-29T02:33:13.063Z" }, ] [[package]] @@ -6108,11 +6120,11 @@ wheels = [ [[package]] name = "oauthlib" -version = "4.0.0" +version = "4.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7a/d8/a1bcc8ba112a627f8ffbdc212a78ce18d3ac07e91a5ca65d27918eee25a1/oauthlib-4.0.0.tar.gz", hash = "sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86", size = 187232, upload-time = "2026-09-28T06:01:18.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/17/2bb072de0b990303c985f26869fd47bb179024a051f47e7f9d208eb92f8e/oauthlib-4.0.1.tar.gz", hash = "sha256:3d73febdb6faa00f277c81fb04be7b7586fb96fc3b7f0b9c772dada69a25ede7", size = 191219, upload-time = "2026-10-31T09:52:58.008Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/df/ef/3b8adde1b32a6f540758bff734d3442c621734c83708e5f02b0ba2f43b84/oauthlib-4.0.1-py3-none-any.whl", hash = "sha256:498a298876e996301950a5b053d12c31628bb7b895db03666430c69ce182eb61", size = 161934, upload-time = "2026-10-31T09:52:56.206Z" }, ] [[package]] @@ -7770,11 +7782,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.4" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, ] [[package]]