From db2f7df17d6e6f1543e9f92e7d9793c49d27bfe0 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Fri, 18 Sep 2026 11:55:32 +0900 Subject: [PATCH 1/6] feat(ci): report each release's schema changes against a rolling upgrade A release does not say what it does to the database. The migrations ship inside litellm-proxy-extras and the first pod that boots applies them with `prisma migrate deploy` while pods on the previous version are still serving traffic; the generated notes list commit titles, where a column addition reads as a feature ("add lifecycle timestamps"). The operator finds out what the upgrade did from their own error rate. #36418 is that failure: `key_type` was added to a table the virtual-key auth path reads with `SELECT v.*`, which invalidated the prepared plans on every pooled connection the old pods held. `ci_cd/migration_impact.py` diffs the migration set between two refs and rates each statement by what it does to a pod still running the base version: breaking (the old pods cannot recover: a column they address is dropped or renamed, or a NOT NULL now rejects their writes), prepared-plan (their cached plans are rejected until the connections are recreated: a whole-row read gains a column, or a column changes type), lock (blocking DDL or a backfill on a table already serving traffic), info. The tables that are read whole-row are discovered from the source tree at --base, the code the old pods are running, rather than hardcoded, so the report keeps working as those queries move. `DO $$ ... $$` bodies are descended into, because that is where renames hide. create-release.yml runs it for the range between the previous stable tag and the release commit and puts the result at the top of the release body, so the answer ships with the release instead of being reconstructed afterwards. The step is continue-on-error: a release is never held up by its own notes. Verified against the range in #36418 (v1.93.0 -> v1.99.0): both migrations that touch LiteLLM_VerificationToken are reported and the v1.93.0 auth query is cited at its line; v1.99.0 -> main surfaces a RENAME COLUMN nested in a DO block. Classifying all 179 migrations in the repo yields 7 breaking, all of them real renames, drops, or SET NOT NULL. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/create-release.yml | 28 +- ci_cd/migration_impact.py | 588 ++++++++++++++++++++ litellm-proxy-extras/migration_runbook.md | 28 + tests/test_litellm/test_migration_impact.py | 377 +++++++++++++ 4 files changed, 1020 insertions(+), 1 deletion(-) create mode 100755 ci_cd/migration_impact.py create mode 100644 tests/test_litellm/test_migration_impact.py diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 0ad84cd3ceb..e8a3f446371 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -35,10 +35,30 @@ jobs: exit 1 fi + # The proxy's migrations ship in litellm-proxy-extras, so nothing in the generated notes + # tells an operator what this release does to their database. Summarize it here instead. + - name: Check out the release commit + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ inputs.commit_hash }} + + - name: Summarize the database schema changes + # Never hold up a release over its own release notes: on failure the section is left out. + continue-on-error: true + env: + COMMIT_HASH: ${{ inputs.commit_hash }} + SCHEMA_SECTION: ${{ runner.temp }}/schema_section.md + run: | + python3 ci_cd/migration_impact.py --head "${COMMIT_HASH}" --output "${SCHEMA_SECTION}" + cat "${SCHEMA_SECTION}" >> "${GITHUB_STEP_SUMMARY}" + - name: Create release env: TAG: ${{ inputs.tag }} COMMIT_HASH: ${{ inputs.commit_hash }} + SCHEMA_SECTION: ${{ runner.temp }}/schema_section.md uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | @@ -68,6 +88,12 @@ jobs: return true; }; + const fs = require('fs'); + const schemaPath = process.env.SCHEMA_SECTION; + const schemaSection = fs.existsSync(schemaPath) + ? `${fs.readFileSync(schemaPath, 'utf8').trim()}\n\n---\n\n` + : ''; + const cosignSection = [ `## Verify Docker Image Signature`, ``, @@ -151,7 +177,7 @@ jobs: tag_name: tag, }); - const updatedBody = cosignSection + (response.data.body ?? ''); + const updatedBody = schemaSection + cosignSection + (response.data.body ?? ''); await github.rest.repos.updateRelease({ owner: context.repo.owner, repo: context.repo.repo, diff --git a/ci_cd/migration_impact.py b/ci_cd/migration_impact.py new file mode 100755 index 00000000000..6268d6939f6 --- /dev/null +++ b/ci_cd/migration_impact.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +"""Report what a version bump does to the proxy's Postgres schema. + +A release does not say what it does to the database. The migrations ship inside the +`litellm-proxy-extras` package (`litellm_proxy_extras/migrations/_/migration.sql`) +and the first pod that boots a new version applies them with `prisma migrate deploy`, while the +generated release notes list commit titles, where a column addition reads as a feature ("add +lifecycle timestamps"). An operator upgrading a multi-replica deployment therefore cannot see, +from the release, which of those migrations the pods still running the previous version survive. + +This script diffs the migration set between two refs, classifies every statement by what it does +to a pod still running the base version, and renders the result as markdown (for the release body) +or JSON. Severities, worst first: + +- `breaking` — the old pods cannot recover on their own: a column they read is gone, renamed + or retyped, or a constraint now rejects the rows they write +- `prepared-plan` — the change alters the result type of a query the old pods have prepared + (a `SELECT .*` gaining a column, or a column changing type), so the + plans cached on their pooled connections are rejected with + `cached plan must not change result type` until those connections are + recreated +- `lock` — the migration takes a blocking lock (or rewrites rows) on a table that is + already serving traffic +- `info` — no effect on the old pods (new tables, and changes to them) + +The tables that are read through `SELECT .*` are discovered from the source tree at `--base` +(the code the old pods are running) rather than hardcoded, so the report keeps working as those +queries move. + + python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 + +Standard library only, so it runs on a bare checkout with no `uv sync`. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +MIGRATIONS_DIR: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations" +# git pathspec wildcards cross directory boundaries, so these reach every file under each tree. +SOURCE_GLOBS: Final = ("litellm/*.py", "enterprise/*.py") +TABLE_MARKER: Final = 'FROM "' +# `*` inside a view is expanded when the view is created, so it holds no per-connection plan. +VIEW_MARKER: Final = re.compile(r"\bCREATE\s+(?:OR\s+REPLACE\s+)?(?:MATERIALIZED\s+)?VIEW\b", re.IGNORECASE) + +BREAKING: Final = "breaking" +PREPARED_PLAN: Final = "prepared-plan" +LOCK: Final = "lock" +INFO: Final = "info" +SEVERITY_ORDER: Final = (BREAKING, PREPARED_PLAN, LOCK, INFO) +SEVERITY_ICON: Final = {BREAKING: "🚨", PREPARED_PLAN: "⚠️", LOCK: "🔒", INFO: "ℹ️"} + +# Words that can follow a table name where an alias would sit, so they are never read as one. +ALIAS_STOPWORDS: Final = frozenset( + { + "as", + "cross", + "except", + "fetch", + "for", + "full", + "group", + "having", + "inner", + "join", + "left", + "limit", + "intersect", + "natural", + "offset", + "on", + "order", + "outer", + "returning", + "right", + "set", + "tablesample", + "union", + "using", + "values", + "where", + "window", + } +) + +_LINE_COMMENT: Final = re.compile(r"--[^\n]*") +_BLOCK_COMMENT: Final = re.compile(r"/\*.*?\*/", re.DOTALL) +_DOLLAR_QUOTED: Final = re.compile(r"\$(?P[A-Za-z_]*)\$(?P.*?)\$(?P=tag)\$", re.DOTALL) + +_CREATE_TABLE: Final = re.compile(r'\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE) +_ALTER_TABLE: Final = re.compile(r'\bALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?"?(?P
\w+)"?', re.IGNORECASE) +_DROP_TABLE: Final = re.compile(r'\bDROP\s+TABLE\s+(?:IF\s+EXISTS\s+)?"?(?P
\w+)"?', re.IGNORECASE) +_CREATE_INDEX: Final = re.compile( + r'\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?"?\w+"?\s+ON\s+(?:ONLY\s+)?"?(?P
\w+)"?', + re.IGNORECASE, +) +# The `SET` is what separates a backfill from the `ON UPDATE CASCADE` of a foreign key. +_UPDATE: Final = re.compile(r'\bUPDATE\s+(?:ONLY\s+)?"?(?P
\w+)"?\s+SET\b', re.IGNORECASE) +_INSERT: Final = re.compile(r'\bINSERT\s+INTO\s+"?(?P
\w+)"?', re.IGNORECASE) + +_DROP_COLUMN: Final = re.compile(r"\bDROP\s+COLUMN\b", re.IGNORECASE) +_ADD_COLUMN: Final = re.compile(r"\bADD\s+COLUMN\b", re.IGNORECASE) +# One ALTER TABLE carries many comma-separated clauses; split only at a clause keyword so the commas +# inside `NUMERIC(10,2)` or `ARRAY[]::TEXT[]` stay put. +_CLAUSE_SPLIT: Final = re.compile(r",\s*(?=(?:ADD|ALTER|DROP)\s+(?:COLUMN|CONSTRAINT)\b)", re.IGNORECASE) +# Column and table renames only; `ALTER INDEX ... RENAME TO` and `RENAME CONSTRAINT` change nothing a pod reads. +_RENAME: Final = re.compile(r"\bRENAME\s+(?:COLUMN\b|TO\b)", re.IGNORECASE) +_ALTER_TYPE: Final = re.compile(r"\bALTER\s+(?:COLUMN\s+)?\S+\s+(?:SET\s+DATA\s+)?TYPE\b", re.IGNORECASE) +_SET_NOT_NULL: Final = re.compile(r"\bSET\s+NOT\s+NULL\b", re.IGNORECASE) +# `DROP NOT NULL` relaxes a column and `IS NOT NULL` is a predicate; neither adds a constraint. +_NOT_NULL: Final = re.compile(r"(?\"{3}|'{3})(?P.*?)(?P=quote)", re.DOTALL) +_FROM_JOIN: Final = re.compile( + r'\b(?:FROM|JOIN)\s+"(?P
\w+)"(?:\s+AS)?(?:\s+(?P[A-Za-z_]\w*))?', re.IGNORECASE +) +_STAR_ALIAS: Final = re.compile(r"\b(?P[A-Za-z_]\w*)\.\*") +_BARE_STAR: Final = re.compile(r"\bSELECT\s+\*", re.IGNORECASE) +# Release tags an operator could actually be running: 1.99.0, v1.99.0, 1.99.0.post1, v1.99.0-stable. +_STABLE_TAG: Final = re.compile(r"^v?\d+\.\d+\.\d+(?:\.post\d+|-stable(?:\.patch\.\d+)?)?$") + + +@dataclass(frozen=True, slots=True) +class StarRead: + """A query that reads a whole table row, i.e. one whose result type follows the schema.""" + + table: str + alias: str + joined: tuple[str, ...] + location: str + + +@dataclass(frozen=True, slots=True) +class Finding: + severity: str + migration: str + table: str + statement: str + effect: str + + def as_dict(self) -> dict[str, str]: + return { + "severity": self.severity, + "migration": self.migration, + "table": self.table, + "statement": self.statement, + "effect": self.effect, + } + + +@dataclass(frozen=True, slots=True) +class Report: + base: str + head: str + migrations: tuple[str, ...] + findings: tuple[Finding, ...] + star_reads: tuple[StarRead, ...] + + @property + def worst(self) -> str | None: + for severity in SEVERITY_ORDER: + if any(finding.severity == severity for finding in self.findings): + return severity + return None + + +class GitError(RuntimeError): + """A git command the report depends on could not be run.""" + + +def _git(repo: Path, *args: str) -> str: + result: Final = subprocess.run(("git", "-C", str(repo), *args), check=False, capture_output=True, text=True) + if result.returncode != 0: + raise GitError(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout + + +def _git_or_none(repo: Path, *args: str) -> str | None: + try: + return _git(repo, *args) + except GitError: + return None + + +def normalize(statement: str) -> str: + return " ".join(statement.split()).strip() + + +def split_statements(sql: str) -> list[str]: + """Split a migration into single-line statements, descending into `DO $$ ... $$` bodies.""" + cleaned: Final = _BLOCK_COMMENT.sub(" ", _LINE_COMMENT.sub(" ", sql)) + bodies: Final[list[str]] = [] + + def stash(match: re.Match[str]) -> str: + bodies.append(match.group("body")) + return " " + + outer: Final = _DOLLAR_QUOTED.sub(stash, cleaned) + statements: Final = [normalize(part) for part in outer.split(";")] + for body in bodies: + statements.extend(split_statements(body)) + return [statement for statement in statements if statement] + + +def statement_table(statement: str) -> str: + for pattern in (_CREATE_TABLE, _ALTER_TABLE, _DROP_TABLE, _CREATE_INDEX, _UPDATE, _INSERT): + match: Final = pattern.search(statement) + if match is not None: + return match.group("table") + return "" + + +def migration_names(repo: Path, ref: str) -> tuple[str, ...]: + """The migration directories at `ref`, skipping `migration_lock.toml` and any stray file.""" + listing: Final = _git(repo, "ls-tree", ref, f"{MIGRATIONS_DIR}/") + names: Final[list[str]] = [] + for line in listing.splitlines(): + # \t + meta, _, path = line.partition("\t") + if meta.split()[1:2] == ["tree"]: + names.append(Path(path).name) + return tuple(sorted(names)) + + +def read_migration(repo: Path, ref: str, name: str) -> str | None: + """The SQL of one migration, or None when the directory has no `migration.sql` (Prisma skips it too).""" + return _git_or_none(repo, "show", f"{ref}:{MIGRATIONS_DIR}/{name}/migration.sql") + + +def tables_created_before(repo: Path, ref: str) -> frozenset[str]: + """Every table some migration at `ref` creates, so a later `CREATE TABLE IF NOT EXISTS` re-declaration + of an existing table is not mistaken for a new one.""" + # Fixed string: `git grep -E` is POSIX ERE, where `\s` is not portable. + listing: Final = _git_or_none( + repo, "grep", "-h", "-i", "--fixed-strings", "CREATE TABLE", ref, "--", f"{MIGRATIONS_DIR}/" + ) + return frozenset(match.group("table") for match in _CREATE_TABLE.finditer(listing or "")) + + +def _alias_map(body: str) -> tuple[dict[str, str], tuple[str, ...]]: + aliases: Final[dict[str, str]] = {} + tables: Final[list[str]] = [] + for match in _FROM_JOIN.finditer(body): + table: Final = match.group("table") + if table not in tables: + tables.append(table) + alias: Final = match.group("alias") + if alias is not None and alias.lower() not in ALIAS_STOPWORDS: + aliases[alias] = table + return aliases, tuple(tables) + + +def star_reads_in_source(text: str, path: str) -> list[StarRead]: + """Find whole-row reads (`SELECT v.*`, `SELECT *`) in the SQL literals of one source file.""" + found: Final[list[StarRead]] = [] + for literal in _SQL_LITERAL.finditer(text): + raw: Final = literal.group("body") + if TABLE_MARKER not in raw or "SELECT" not in raw.upper(): + continue + if VIEW_MARKER.search(raw) is not None: + continue + # `-- Added comma to separate b.* columns` is a comment, not a read. + body: Final = _LINE_COMMENT.sub(" ", raw) + aliases, tables = _alias_map(body) + line: Final = text.count("\n", 0, literal.start()) + 1 + location: Final = f"{path}:{line}" + starred: Final[list[tuple[str, str]]] = [ + (aliases[match.group("alias")], match.group("alias")) + for match in _STAR_ALIAS.finditer(body) + if match.group("alias") in aliases + ] + if not starred and _BARE_STAR.search(body) is not None and len(tables) == 1: + starred.append((tables[0], "*")) + for table, alias in dict.fromkeys(starred): + found.append( + StarRead( + table=table, + alias=alias, + joined=tuple(other for other in tables if other != table), + location=location, + ) + ) + return found + + +def discover_star_reads(repo: Path, ref: str) -> tuple[StarRead, ...]: + listing: Final = _git_or_none(repo, "grep", "-l", "--fixed-strings", TABLE_MARKER, ref, "--", *SOURCE_GLOBS) + if listing is None: + return () + found: Final[list[StarRead]] = [] + for line in listing.splitlines(): + path: Final = line.split(":", 1)[1] if ":" in line else line + text: Final = _git_or_none(repo, "show", f"{ref}:{path}") + if text is not None: + found.extend(star_reads_in_source(text, path)) + return tuple(found) + + +def _shape(read: StarRead) -> str: + return "SELECT *" if read.alias == "*" else f"SELECT {read.alias}.*" + + +def _star_read_for(table: str, star_reads: Sequence[StarRead]) -> StarRead | None: + matches: Final = [read for read in star_reads if read.table == table] + return max(matches, key=lambda read: len(read.joined)) if matches else None + + +def _joined_read_for(table: str, star_reads: Sequence[StarRead]) -> StarRead | None: + return next((read for read in star_reads if table in read.joined), None) + + +def classify( + migration: str, + statement: str, + fresh_tables: frozenset[str], + star_reads: Sequence[StarRead], +) -> Finding: + """Rate one statement by what it does to a pod still running the base version.""" + table: Final = statement_table(statement) + + def finding(severity: str, effect: str) -> Finding: + return Finding(severity=severity, migration=migration, table=table, statement=statement, effect=effect) + + if _CREATE_TABLE.search(statement) is not None: + return finding(INFO, "New table. The previous version does not read or write it.") + if table and table in fresh_tables: + return finding(INFO, "Table is created by this same upgrade, so no old pod reads or writes it yet.") + + star: Final = _star_read_for(table, star_reads) + where: Final = f" (`{star.location}`)" if star is not None else "" + + if _DROP_COLUMN.search(statement) is not None or _DROP_TABLE.search(statement) is not None: + return finding( + BREAKING, + f"Removes something the previous version still reads{where}. Old pods keep failing until they are gone; " + "reconnecting does not help.", + ) + if _ALTER_TABLE.search(statement) is not None and _RENAME.search(statement) is not None: + return finding( + BREAKING, + f"Renames something the previous version still addresses by its old name{where}. Old pods cannot " + "recover on their own.", + ) + if _SET_NOT_NULL.search(statement) is not None: + return finding(BREAKING, "Rows the previous version writes without this column are now rejected.") + if _ADD_COLUMN.search(statement) is not None and _adds_not_null_without_default(statement): + return finding(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") + if _ALTER_TYPE.search(statement) is not None: + return finding( + PREPARED_PLAN, + "Changes a column's type, so every prepared query on the previous version that returns it is rejected " + "with `cached plan must not change result type` until its connection is recreated. If the new type " + "is one the old client cannot map, it keeps failing after that.", + ) + if _ADD_COLUMN.search(statement) is not None and star is not None: + return finding( + PREPARED_PLAN, + f"Read whole-row by `{_shape(star)}`{where}, so the prepared plans cached on the previous version's " + "pooled connections are rejected with `cached plan must not change result type` until those " + "connections are recreated.", + ) + if _CREATE_INDEX.search(statement) is not None and _CONCURRENTLY.search(statement) is None: + joined: Final = _joined_read_for(table, star_reads) + read_note: Final = ( + f" The table is joined by `{_shape(joined)}` (`{joined.location}`), so that read waits too." + if joined is not None + else "" + ) + return finding( + LOCK, + "Blocks writes to the table (reads continue) until the index is built; the wait scales with the " + f"table's size.{read_note}", + ) + if _ADD_CONSTRAINT.search(statement) is not None and _NOT_VALID.search(statement) is None: + return finding(LOCK, "Validates the constraint against every existing row while holding the table lock.") + if _UPDATE.search(statement) is not None: + return finding(LOCK, "Rewrites existing rows, so it holds row locks for as long as the backfill runs.") + if _ADD_COLUMN.search(statement) is not None: + return finding(INFO, "Adds a nullable column to a table the previous version does not read whole-row.") + if _INSERT.search(statement) is not None: + return finding(INFO, "Inserts rows. The previous version is not affected.") + return finding(INFO, "No effect on the previous version.") + + +def _adds_not_null_without_default(statement: str) -> bool: + """True when any `ADD COLUMN` clause of the statement is NOT NULL and has no DEFAULT of its own.""" + return any( + _NOT_NULL.search(clause) is not None and _DEFAULT.search(clause) is None + for clause in _CLAUSE_SPLIT.split(statement) + if _ADD_COLUMN.search(clause) is not None + ) + + +def build_report(repo: Path, base: str, head: str) -> Report: + base_names: Final = set(migration_names(repo, base)) + head_names: Final = migration_names(repo, head) + new_names: Final = tuple(name for name in head_names if name not in base_names) + # The plans that break are the ones the old pods hold, so the queries that matter are the base version's. + star_reads: Final = discover_star_reads(repo, base) + + sql_by_migration: Final = {name: sql for name in new_names if (sql := read_migration(repo, head, name)) is not None} + created_here: Final = frozenset( + match.group("table") + for sql in sql_by_migration.values() + for statement in split_statements(sql) + if (match := _CREATE_TABLE.search(statement)) is not None + ) + # A re-declared `CREATE TABLE IF NOT EXISTS` of a table the old pods already use is not a new table. + fresh_tables: Final = created_here - tables_created_before(repo, base) + + findings: Final[list[Finding]] = [] + for name, sql in sql_by_migration.items(): + for statement in split_statements(sql): + findings.append(classify(name, statement, fresh_tables, star_reads)) + + ranked: Final = sorted(findings, key=lambda finding: (SEVERITY_ORDER.index(finding.severity), finding.migration)) + return Report(base=base, head=head, migrations=new_names, findings=tuple(ranked), star_reads=star_reads) + + +PROCEDURE: Final = { + BREAKING: ( + "**Do not run the old and new versions side by side.** Take the previous version out of service before the " + "new version applies these migrations, or split the change across two releases." + ), + PREPARED_PLAN: ( + "Expect the queries cited above to fail on pods still running the previous version until their pooled " + "connections are recreated. LiteLLM recreates them on the first such failure as of `1.98.0`; on older " + "versions, or to avoid the failures entirely, set `general_settings.database_disable_prepared_statements: " + "true` before upgrading, or drain the old pods before the new ones migrate." + ), + LOCK: "Apply these migrations in a maintenance window, or during a traffic trough, and watch for lock waits.", + INFO: "Safe to apply during a rolling upgrade.", +} + + +def _cited_reads(report: Report) -> tuple[StarRead, ...]: + """The whole-row reads that actually bear on this release, one per table.""" + tables: Final = {finding.table for finding in report.findings if finding.severity != INFO} + representatives: Final = (_star_read_for(table, report.star_reads) for table in sorted(tables)) + return tuple(read for read in representatives if read is not None) + + +def render_markdown(report: Report) -> str: + lines: Final[list[str]] = ["## Database Schema Changes", ""] + if not report.migrations: + lines.append(f"No database schema changes since `{report.base}`.") + return "\n".join(lines) + "\n" + + count: Final = len(report.migrations) + plural: Final = "" if count == 1 else "s" + lines.extend( + [ + f"This release adds **{count} migration{plural}** to `litellm-proxy-extras`, applied by the first pod " + f"that boots (`prisma migrate deploy`). Rated against a pod still running `{report.base}`:", + "", + "| | Migration | Statement | Effect on pods still on the previous version |", + "|:--:|:--|:--|:--|", + ] + ) + notable: Final = [finding for finding in report.findings if finding.severity != INFO] + for finding in notable[:MAX_ROWS]: + lines.append( + f"| {SEVERITY_ICON[finding.severity]} | `{finding.migration}` | `{_clip(finding.statement)}` | " + f"{finding.effect} |" + ) + if len(notable) > MAX_ROWS: + lines.append( + f"| | | | …and {len(notable) - MAX_ROWS} more, worst first. Run " + f"`python3 ci_cd/migration_impact.py --base {report.base} --head {report.head}` for the full list. |" + ) + + worst: Final = report.worst + if worst is None or worst == INFO: + lines.append("| ℹ️ | — | — | Nothing in this release affects a pod running the previous version. |") + lines.extend(["", PROCEDURE[worst or INFO], ""]) + cited: Final = _cited_reads(report) + if cited: + shapes: Final = ", ".join(sorted({f"`{_shape(read)}` on `{read.table}` (`{read.location}`)" for read in cited})) + lines.append(f"Whole-row reads considered: {shapes}.") + lines.append("") + return "\n".join(lines) + + +_DDL_START: Final = re.compile( + r"\b(?:ALTER\s+TABLE|CREATE\s+(?:UNIQUE\s+)?INDEX|CREATE\s+TABLE|DROP\s+TABLE|DROP\s+INDEX|UPDATE|INSERT\s+INTO)\b", + re.IGNORECASE, +) +MAX_ROWS: Final = 20 + + +def _clip(statement: str, limit: int = 90) -> str: + """Render a statement for the table: the DDL itself, clipped, without its `DO $$` preamble.""" + match: Final = _DDL_START.search(statement) + ddl: Final = statement[match.start() :] if match is not None else statement + clipped: Final = ddl if len(ddl) <= limit else f"{ddl[: limit - 1]}…" + return clipped.replace("|", "\\|") + + +def render_json(report: Report) -> str: + payload: Final = { + "base": report.base, + "head": report.head, + "worst_severity": report.worst, + "migrations": list(report.migrations), + "findings": [finding.as_dict() for finding in report.findings], + "star_reads": [ + {"table": read.table, "alias": read.alias, "joined": list(read.joined), "location": read.location} + for read in report.star_reads + ], + } + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def default_base(repo: Path, head: str) -> str: + """The newest stable release reachable from `head`, which is what an operator upgrades from. + + Pre-release tags are skipped: a release cut on top of `v1.102.0-rc.2` still has to tell the + operator what changed since the last version they could have been running. + """ + # Version order, not date order: a backport release branch reaches only its own line's tags, + # and two tags cut in the same second sort arbitrarily by date. + listing: Final = _git_or_none(repo, "tag", "--merged", head, "--sort=-v:refname") + head_sha: Final = (_git_or_none(repo, "rev-parse", head) or "").strip() + for line in (listing or "").splitlines(): + tag = line.strip() + if not _STABLE_TAG.match(tag): + continue + tagged = (_git_or_none(repo, "rev-list", "-n", "1", tag) or "").strip() + if tagged and tagged != head_sha: + return tag + described: Final = _git_or_none(repo, "describe", "--tags", "--abbrev=0", f"{head}^") + if described is None or not described.strip(): + raise GitError(f"found no release tag before {head}; pass --base explicitly (the checkout may lack tags)") + return described.strip() + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser: Final = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base", help="ref of the version being upgraded FROM (default: the tag before --head)") + parser.add_argument("--head", default="HEAD", help="ref of the version being upgraded TO (default: HEAD)") + parser.add_argument("--repo", default=str(Path(__file__).resolve().parents[1]), help="path to the repository") + parser.add_argument("--format", choices=("markdown", "json"), default="markdown") + parser.add_argument("--output", help="write the report here instead of stdout") + parser.add_argument( + "--fail-on", + choices=(BREAKING, PREPARED_PLAN, LOCK), + help="exit 1 when a finding at this severity or worse is present", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args: Final = parse_args(argv) + repo: Final = Path(args.repo) + try: + base: Final = args.base or default_base(repo, args.head) + report: Final = build_report(repo, base, args.head) + except GitError as error: + sys.stderr.write(f"migration impact report skipped: {error}\n") + return 2 + + rendered: Final = render_markdown(report) if args.format == "markdown" else render_json(report) + if args.output: + Path(args.output).write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + + worst: Final = report.worst + if args.fail_on and worst is not None and SEVERITY_ORDER.index(worst) <= SEVERITY_ORDER.index(args.fail_on): + sys.stderr.write(f"migration impact: found {worst} changes between {base} and {args.head}\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/litellm-proxy-extras/migration_runbook.md b/litellm-proxy-extras/migration_runbook.md index b1e9236e520..39cad08b303 100644 --- a/litellm-proxy-extras/migration_runbook.md +++ b/litellm-proxy-extras/migration_runbook.md @@ -121,6 +121,34 @@ rm -rf litellm-proxy-extras/litellm_proxy_extras/migrations/[empty_dir] - Never edit existing migration files - Commit schema + migration together +## Upgrade Impact Report + +Migrations in this package are invisible to an operator reading a release: the notes list commit +titles, and the first pod to boot applies the migrations with `prisma migrate deploy` while the +pods running the previous version are still serving traffic. `ci_cd/migration_impact.py` diffs the +migration set between two refs and rates every statement by what it does to those older pods: + +| Severity | Meaning | +|---|---| +| `breaking` | The old pods cannot recover on their own — a column they read is gone, renamed or retyped, or a constraint now rejects the rows they write | +| `prepared-plan` | The change alters the result type of a query the old pods have prepared - a `SELECT .*` gaining a column, or a column changing type - so the plans cached on their pooled connections are rejected with `cached plan must not change result type` until those connections are recreated ([#36418](https://github.com/BerriAI/litellm/issues/36418)) | +| `lock` | The migration takes a blocking lock, or rewrites rows, on a table already serving traffic | +| `info` | No effect on the old pods (new tables, and changes to them) | + +The tables read whole-row are discovered from the source tree at `--base` - the code the old pods +are running - not hardcoded, so the report follows those queries as they move. + +`create-release.yml` runs this and puts the result at the top of the release body. To see it for +any pair of versions yourself: + +```bash +python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 +python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 --format json +``` + +Standard library only, so it needs no `uv sync`. `--fail-on breaking` exits non-zero when a +migration in the range is not safe to apply under a rolling upgrade. + --- **Done with migration?** See [build_and_publish.md](./build_and_publish.md) to publish a new `litellm-proxy-extras` package. diff --git a/tests/test_litellm/test_migration_impact.py b/tests/test_litellm/test_migration_impact.py new file mode 100644 index 00000000000..c23cd3627e7 --- /dev/null +++ b/tests/test_litellm/test_migration_impact.py @@ -0,0 +1,377 @@ +"""Tests for ci_cd/migration_impact.py. + +The fixtures are the real statements from the migrations that caused the rolling-upgrade +failures this report exists to announce: the `key_type` column added to +`LiteLLM_VerificationToken` (issue #36418) and the `api_key_id` -> `target_id` rename hidden +inside a `DO $$` block. +""" + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType +from typing import Final + +ROOT: Final = Path(__file__).resolve().parents[2] +CI_CD: Final = ROOT / "ci_cd" + + +def _load(name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, CI_CD / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +impact: Final = _load("migration_impact") + +AUTH_QUERY_SOURCE: Final = ''' + sql_query = """ + SELECT + v.*, + t.spend AS team_spend, + -- Added comma to separate b.* columns + b.max_budget AS litellm_budget_table_max_budget + FROM "LiteLLM_VerificationToken" AS v + LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id + LEFT JOIN "LiteLLM_BudgetTable" AS b ON v.budget_id = b.budget_id + WHERE v.token = $1 + """ +''' + +VIEW_SOURCE: Final = ''' + await db.execute_raw(""" + CREATE OR REPLACE VIEW "LiteLLM_VerificationTokenView" AS + SELECT v.*, t.spend AS team_spend + FROM "LiteLLM_VerificationToken" v + LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id; + """) +''' + +RENAME_IN_DO_BLOCK: Final = """ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id' + ) THEN + ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id"; + END IF; +END $$; + +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key'; +""" + + +def _auth_read() -> object: + reads = impact.star_reads_in_source(AUTH_QUERY_SOURCE, "litellm/proxy/utils.py") + assert len(reads) == 1 + return reads[0] + + +def _classify(statement: str, fresh: frozenset[str] = frozenset(), reads: tuple[object, ...] = ()) -> object: + return impact.classify("20260713230852_add_key_type", statement, fresh, reads) + + +def test_split_statements_descends_into_dollar_quoted_blocks() -> None: + statements = impact.split_statements(RENAME_IN_DO_BLOCK) + renames = [statement for statement in statements if "RENAME COLUMN" in statement] + assert len(renames) == 1, statements + assert '"api_key_id" TO "target_id"' in renames[0] + + +def test_split_statements_drops_comments() -> None: + statements = impact.split_statements('-- add a column\nALTER TABLE "T" ADD COLUMN "c" TEXT; /* trailing */') + assert statements == ['ALTER TABLE "T" ADD COLUMN "c" TEXT'] + + +def test_star_read_discovery_records_table_alias_and_joins() -> None: + read = _auth_read() + assert read.table == "LiteLLM_VerificationToken" + assert read.alias == "v" + assert set(read.joined) == {"LiteLLM_TeamTable", "LiteLLM_BudgetTable"} + assert read.location.startswith("litellm/proxy/utils.py:") + + +def test_star_read_discovery_ignores_aliases_that_only_appear_in_a_comment() -> None: + # `-- Added comma to separate b.* columns` must not register LiteLLM_BudgetTable as whole-row read. + tables = {read.table for read in impact.star_reads_in_source(AUTH_QUERY_SOURCE, "litellm/proxy/utils.py")} + assert tables == {"LiteLLM_VerificationToken"} + + +def test_star_read_discovery_ignores_view_definitions() -> None: + # A view expands `*` when it is created, so it holds no per-connection prepared plan. + assert impact.star_reads_in_source(VIEW_SOURCE, "litellm/proxy/db/create_views.py") == [] + + +def test_add_column_on_a_whole_row_read_table_is_a_prepared_plan_change() -> None: + finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) + assert finding.severity == impact.PREPARED_PLAN + assert finding.table == "LiteLLM_VerificationToken" + assert "cached plan must not change result type" in finding.effect + assert "litellm/proxy/utils.py" in finding.effect + + +def test_add_column_elsewhere_is_not_reported() -> None: + finding = _classify('ALTER TABLE "LiteLLM_AuditLog" ADD COLUMN "note" TEXT', reads=(_auth_read(),)) + assert finding.severity == impact.INFO + + +def test_drop_column_outranks_the_prepared_plan_rule() -> None: + finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_type"', reads=(_auth_read(),)) + assert finding.severity == impact.BREAKING + + +def test_rename_is_breaking() -> None: + finding = _classify('ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id"') + assert finding.severity == impact.BREAKING + + +def test_not_null_column_is_breaking_only_without_a_default() -> None: + without_default = _classify('ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL') + with_default = _classify( + 'ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' + ) + assert without_default.severity == impact.BREAKING + assert with_default.severity == impact.INFO + + +def test_index_builds_are_rated_by_whether_they_lock() -> None: + blocking = _classify('CREATE INDEX "idx" ON "LiteLLM_SpendLogs"("api_key")') + concurrent = _classify('CREATE INDEX CONCURRENTLY "idx" ON "LiteLLM_SpendLogs"("api_key")') + assert blocking.severity == impact.LOCK + assert concurrent.severity == impact.INFO + + +def test_unvalidated_constraint_is_not_a_lock() -> None: + validating = _classify('ALTER TABLE "LiteLLM_SpendLogs" ADD CONSTRAINT "fk" FOREIGN KEY ("t") REFERENCES "T"("id")') + deferred = _classify( + 'ALTER TABLE "LiteLLM_SpendLogs" ADD CONSTRAINT "fk" FOREIGN KEY ("t") REFERENCES "T"("id") NOT VALID' + ) + assert validating.severity == impact.LOCK + assert deferred.severity == impact.INFO + + +def test_changes_to_a_table_created_by_the_same_upgrade_are_not_reported() -> None: + finding = _classify( + 'CREATE INDEX "idx" ON "LiteLLM_NewTable"("id")', + fresh=frozenset({"LiteLLM_NewTable"}), + ) + assert finding.severity == impact.INFO + assert "created by this same upgrade" in finding.effect + + +def _report(findings: tuple[object, ...], migrations: tuple[str, ...] = ("20260713230852_add_key_type",)) -> object: + return impact.Report( + base="v1.93.0", head="v1.99.0", migrations=migrations, findings=findings, star_reads=(_auth_read(),) + ) + + +def test_markdown_for_an_upgrade_with_no_migrations() -> None: + rendered = impact.render_markdown(_report(findings=(), migrations=())) + assert "No database schema changes since `v1.93.0`." in rendered + assert "|" not in rendered + + +def test_markdown_leads_with_the_worst_finding_and_its_procedure() -> None: + finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) + rendered = impact.render_markdown(_report(findings=(finding,))) + assert "20260713230852_add_key_type" in rendered + assert impact.SEVERITY_ICON[impact.PREPARED_PLAN] in rendered + assert impact.PROCEDURE[impact.PREPARED_PLAN] in rendered + assert "`SELECT v.*` on `LiteLLM_VerificationToken`" in rendered + + +def test_markdown_caps_the_table_and_says_where_the_rest_is() -> None: + finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) + rendered = impact.render_markdown(_report(findings=tuple([finding] * (impact.MAX_ROWS + 3)))) + assert rendered.count("| ⚠️ |") == impact.MAX_ROWS + assert "…and 3 more" in rendered + + +def test_worst_severity_ranks_breaking_above_the_rest() -> None: + lock = _classify('CREATE INDEX "idx" ON "LiteLLM_SpendLogs"("api_key")') + breaking = _classify('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_type"', reads=(_auth_read(),)) + assert _report(findings=(lock, breaking)).worst == impact.BREAKING + assert _report(findings=(lock,)).worst == impact.LOCK + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(("git", "-C", str(repo), *args), check=True, capture_output=True) + + +def _commit(repo: Path, message: str) -> None: + (repo / "file.txt").write_text(message, encoding="utf-8") + _git(repo, "add", "file.txt") + _git(repo, "commit", "-m", message) + + +def test_default_base_is_the_newest_stable_release_not_the_newest_tag(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "test@example.com") + _git(repo, "config", "user.name", "test") + _commit(repo, "first") + _git(repo, "tag", "v1.98.0") + _commit(repo, "second") + _git(repo, "tag", "v1.99.0") + _commit(repo, "third") + _git(repo, "tag", "v1.100.0-rc.1") + _commit(repo, "fourth") + + # The release being cut sits on top of an rc tag; the operator is upgrading from v1.99.0. + assert impact.default_base(repo, "HEAD") == "v1.99.0" + + +def test_default_base_skips_the_tag_that_points_at_the_release_itself(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "test@example.com") + _git(repo, "config", "user.name", "test") + _commit(repo, "first") + _git(repo, "tag", "v1.99.0") + _commit(repo, "second") + _git(repo, "tag", "v1.100.0") + + assert impact.default_base(repo, "v1.100.0") == "v1.99.0" + + +def test_foreign_key_on_update_is_not_a_backfill() -> None: + # `ON UPDATE CASCADE` is part of the constraint; only `UPDATE ... SET` rewrites rows. + finding = _classify( + 'ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "fk" FOREIGN KEY ("token") ' + 'REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE CASCADE ON UPDATE CASCADE NOT VALID' + ) + assert finding.severity == impact.INFO + backfill = _classify('UPDATE "LiteLLM_SpendLogs" SET "created_at" = "startTime" WHERE "created_at" IS NULL') + assert backfill.severity == impact.LOCK + + +def test_index_rename_is_not_breaking() -> None: + finding = _classify('ALTER INDEX "old_idx" RENAME TO "new_idx"') + assert finding.severity == impact.INFO + + +def test_type_change_is_a_prepared_plan_change_not_a_permanent_break() -> None: + finding = _classify('ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT') + assert finding.severity == impact.PREPARED_PLAN + assert "cached plan must not change result type" in finding.effect + + +def test_not_null_is_judged_per_add_column_clause() -> None: + one_default_is_not_enough = _classify( + 'ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "a" TEXT NOT NULL DEFAULT \'x\', ADD COLUMN "b" TEXT NOT NULL' + ) + assert one_default_is_not_enough.severity == impact.BREAKING + + +def test_drop_not_null_beside_an_add_column_is_not_breaking() -> None: + # 20250707230009_add_mcp_namespaced_tool_name: the `NOT NULL` here belongs to `DROP NOT NULL`. + finding = _classify( + 'ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "mcp_namespaced_tool_name" TEXT, ' + 'ALTER COLUMN "model" DROP NOT NULL' + ) + assert finding.severity == impact.INFO + + +def test_insert_is_informational() -> None: + finding = _classify('INSERT INTO "LiteLLM_Config" ("param_name") VALUES (\'x\')') + assert finding.severity == impact.INFO + + +def test_markdown_escapes_pipes_inside_a_statement() -> None: + finding = _classify("UPDATE \"LiteLLM_SpendLogs\" SET \"note\" = 'a' || 'b'") + rendered = impact.render_markdown(_report(findings=(finding,))) + assert "'a' \\|\\| 'b'" in rendered + assert "'a' || 'b'" not in rendered + + +BASE_SOURCE: Final = ''' +SQL = """ + SELECT v.*, t.spend AS team_spend + FROM "LiteLLM_VerificationToken" AS v + LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id + WHERE v.token = $1 +""" +''' + +HEAD_SOURCE: Final = ''' +SQL = """ + SELECT v.token, v.key_type, t.spend AS team_spend + FROM "LiteLLM_VerificationToken" AS v + LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id + WHERE v.token = $1 +""" +''' + + +def _write_migration(repo: Path, name: str, sql: str) -> None: + directory = repo / impact.MIGRATIONS_DIR / name + directory.mkdir(parents=True) + (directory / "migration.sql").write_text(sql, encoding="utf-8") + + +def _two_version_repo(tmp_path: Path) -> Path: + """A repo whose base tag reads the token table whole-row and whose head no longer does.""" + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "test@example.com") + _git(repo, "config", "user.name", "test") + (repo / "litellm" / "proxy").mkdir(parents=True) + (repo / "litellm" / "proxy" / "utils.py").write_text(BASE_SOURCE, encoding="utf-8") + _write_migration( + repo, + "20250101000000_baseline", + 'CREATE TABLE "LiteLLM_VerificationToken" ("token" TEXT PRIMARY KEY);\n' + 'CREATE TABLE "LiteLLM_TeamTable" ("team_id" TEXT PRIMARY KEY);\n', + ) + (repo / impact.MIGRATIONS_DIR / "migration_lock.toml").write_text('provider = "postgresql"\n', encoding="utf-8") + _git(repo, "add", ".") + _git(repo, "commit", "-q", "-m", "base") + _git(repo, "tag", "v1.0.0") + + (repo / "litellm" / "proxy" / "utils.py").write_text(HEAD_SOURCE, encoding="utf-8") + _write_migration( + repo, + "20250201000000_add_key_type", + # Re-declaring an existing table must not make it "fresh"; the ADD COLUMN after it is real. + 'CREATE TABLE IF NOT EXISTS "LiteLLM_VerificationToken" ("token" TEXT PRIMARY KEY);\n' + 'ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT;\n' + 'CREATE TABLE "LiteLLM_NewTable" ("id" TEXT PRIMARY KEY);\n' + 'CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id");\n', + ) + _git(repo, "add", ".") + _git(repo, "commit", "-q", "-m", "head") + _git(repo, "tag", "v1.1.0") + return repo + + +def test_report_rates_migrations_against_the_queries_the_old_pods_run(tmp_path: Path) -> None: + repo = _two_version_repo(tmp_path) + report = impact.build_report(repo, "v1.0.0", "v1.1.0") + + assert report.migrations == ("20250201000000_add_key_type",) + # The whole-row read exists only at v1.0.0; that is the version whose plans go stale. + assert [read.table for read in report.star_reads] == ["LiteLLM_VerificationToken"] + assert report.star_reads[0].location.startswith("litellm/proxy/utils.py:") + + by_statement = {finding.statement: finding for finding in report.findings} + add_column = by_statement['ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT'] + assert add_column.severity == impact.PREPARED_PLAN + new_index = by_statement['CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id")'] + assert new_index.severity == impact.INFO + assert report.worst == impact.PREPARED_PLAN + + +def test_report_for_the_same_ref_twice_has_nothing_to_say(tmp_path: Path) -> None: + repo = _two_version_repo(tmp_path) + report = impact.build_report(repo, "v1.1.0", "v1.1.0") + assert report.migrations == () + assert report.findings == () + assert "No database schema changes since `v1.1.0`." in impact.render_markdown(report) From 886b59b56db36f1f5ce4b7e1b4eba858d888ce3d Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Fri, 18 Sep 2026 12:46:32 +0900 Subject: [PATCH 2/6] fix(ci): rate drops, renames, types and constraints on what the old client uses Review on #41744 found the ratings asserted more than they knew: every drop, rename and NOT NULL was breaking whether or not the old pods touched the column, a narrowing type change was rated as recoverable, and a new constraint could land as info because NOT VALID was read as "not enforced" when it only skips the scan of existing rows. The evidence now comes from the base version itself. schema.prisma at --base lists every column the old Prisma client selects and inserts, so dropping or renaming a column that schema never declared is info, a SET NOT NULL on a column the old client already always writes is info, and a type change is prepared-plan only when the new Postgres type is one the old Prisma type can still read (Int to BIGINT); anything else is breaking. Constraints get their own severity, write-reject, between breaking and prepared-plan: the old pods never enforced the rule, so rows they write may now be rejected, which is conditional on data rather than certain. A unique index that replaces one dropped in the same migration is only a lock. Without a schema at --base the old behaviour stands and the footnote says so. classify is split into ordered rule functions to stay under the complexity gate. Severity labels are text instead of emoji and the em dashes are gone, per the repository style rules. Verified on the full migration history: 7 breaking, all real. v1.99.0 to main adds one write-reject, the cascade foreign key on LiteLLM_JWTKeyMapping.token. Co-Authored-By: Claude Fable 5.1 --- ci_cd/migration_impact.py | 444 ++++++++++++++++---- litellm-proxy-extras/migration_runbook.md | 16 +- tests/test_litellm/test_migration_impact.py | 216 +++++++++- 3 files changed, 583 insertions(+), 93 deletions(-) diff --git a/ci_cd/migration_impact.py b/ci_cd/migration_impact.py index 6268d6939f6..9211e3314f5 100755 --- a/ci_cd/migration_impact.py +++ b/ci_cd/migration_impact.py @@ -12,20 +12,21 @@ This script diffs the migration set between two refs, classifies every statement to a pod still running the base version, and renders the result as markdown (for the release body) or JSON. Severities, worst first: -- `breaking` — the old pods cannot recover on their own: a column they read is gone, renamed - or retyped, or a constraint now rejects the rows they write -- `prepared-plan` — the change alters the result type of a query the old pods have prepared - (a `SELECT .*` gaining a column, or a column changing type), so the - plans cached on their pooled connections are rejected with - `cached plan must not change result type` until those connections are - recreated -- `lock` — the migration takes a blocking lock (or rewrites rows) on a table that is - already serving traffic -- `info` — no effect on the old pods (new tables, and changes to them) +- `breaking`: the old pods cannot recover on their own. A column they select is dropped, renamed + or changed to a type their client cannot read, or a NOT NULL now rejects the rows they write +- `write-reject`: a new constraint the old pods never enforced. Rows they write that violate it + are rejected; whether any do depends on the data +- `prepared-plan`: the result type of a query the old pods have prepared changes (a whole-row + `SELECT .*` gains a column, or a column widens), so the plans cached on their pooled + connections are rejected with `cached plan must not change result type` until those + connections are recreated +- `lock`: the migration takes a blocking lock, or rewrites rows, on a table already serving traffic +- `info`: no effect on the old pods (new tables, changes to them, columns they do not know) -The tables that are read through `SELECT .*` are discovered from the source tree at `--base` -(the code the old pods are running) rather than hardcoded, so the report keeps working as those -queries move. +The evidence for what the old pods read and write is the base version itself: `schema.prisma` at +`--base` lists every column the old Prisma client selects and inserts, and the source tree at +`--base` holds the raw `SELECT .*` queries. Nothing is hardcoded, so the report keeps +working as the schema and those queries move. python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 @@ -52,11 +53,34 @@ TABLE_MARKER: Final = 'FROM "' VIEW_MARKER: Final = re.compile(r"\bCREATE\s+(?:OR\s+REPLACE\s+)?(?:MATERIALIZED\s+)?VIEW\b", re.IGNORECASE) BREAKING: Final = "breaking" +WRITE_REJECT: Final = "write-reject" PREPARED_PLAN: Final = "prepared-plan" LOCK: Final = "lock" INFO: Final = "info" -SEVERITY_ORDER: Final = (BREAKING, PREPARED_PLAN, LOCK, INFO) -SEVERITY_ICON: Final = {BREAKING: "🚨", PREPARED_PLAN: "⚠️", LOCK: "🔒", INFO: "ℹ️"} +SEVERITY_ORDER: Final = (BREAKING, WRITE_REJECT, PREPARED_PLAN, LOCK, INFO) +SEVERITY_LABEL: Final = { + BREAKING: "BREAKING", + WRITE_REJECT: "WRITE-REJECT", + PREPARED_PLAN: "PREPARED-PLAN", + LOCK: "LOCK", + INFO: "INFO", +} +SCHEMA_PATHS: Final = ("schema.prisma", "litellm/proxy/schema.prisma") +PRISMA_SCALARS: Final = frozenset( + {"String", "Int", "BigInt", "Float", "Decimal", "Boolean", "DateTime", "Json", "Bytes"} +) +# Postgres types a column of each Prisma type can become and still be read by the old client. +# Anything else is rated breaking: the plan is rejected once, and the retry cannot map the value. +WIDENING: Final = { + "Int": frozenset({"INT", "INT4", "INTEGER", "BIGINT", "INT8", "NUMERIC", "DECIMAL", "DOUBLE", "REAL", "FLOAT"}), + "BigInt": frozenset({"BIGINT", "INT8", "NUMERIC", "DECIMAL"}), + "Float": frozenset({"DOUBLE", "REAL", "FLOAT", "NUMERIC", "DECIMAL"}), + "Decimal": frozenset({"NUMERIC", "DECIMAL"}), + "String": frozenset({"TEXT", "VARCHAR", "CHARACTER", "CHAR", "CITEXT"}), + "DateTime": frozenset({"TIMESTAMP", "TIMESTAMPTZ"}), + "Json": frozenset({"JSON", "JSONB"}), + "Boolean": frozenset({"BOOLEAN", "BOOL"}), +} # Words that can follow a table name where an alias would sit, so they are never read as one. ALIAS_STOPWORDS: Final = frozenset( @@ -121,6 +145,36 @@ _DEFAULT: Final = re.compile(r"\bDEFAULT\b", re.IGNORECASE) _CONCURRENTLY: Final = re.compile(r"\bCONCURRENTLY\b", re.IGNORECASE) _ADD_CONSTRAINT: Final = re.compile(r"\bADD\s+CONSTRAINT\b", re.IGNORECASE) _NOT_VALID: Final = re.compile(r"\bNOT\s+VALID\b", re.IGNORECASE) +_UNIQUE_INDEX: Final = re.compile(r"\bCREATE\s+UNIQUE\s+INDEX\b", re.IGNORECASE) + +_DROP_COLUMN_NAME: Final = re.compile(r'\bDROP\s+COLUMN\s+(?:IF\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE) +_RENAME_COLUMN: Final = re.compile(r'\bRENAME\s+COLUMN\s+"?(?P\w+)"?\s+TO\b', re.IGNORECASE) +_RENAME_TABLE: Final = re.compile(r"\bRENAME\s+TO\b", re.IGNORECASE) +_SET_NOT_NULL_COLUMN: Final = re.compile( + r'\bALTER\s+(?:COLUMN\s+)?"?(?P\w+)"?\s+SET\s+NOT\s+NULL', re.IGNORECASE +) +_TYPE_CHANGE: Final = re.compile( + r'\bALTER\s+(?:COLUMN\s+)?"?(?P\w+)"?\s+(?:SET\s+DATA\s+)?TYPE\s+(?P\w+)', re.IGNORECASE +) +_CONSTRAINT_COLUMNS: Final = re.compile( + r"\b(?:FOREIGN\s+KEY|UNIQUE|PRIMARY\s+KEY)\s*\((?P[^)]*)\)", re.IGNORECASE +) +_CHECK_BODY: Final = re.compile(r"\bCHECK\s*\((?P.*)\)", re.IGNORECASE | re.DOTALL) +_INDEX_COLUMNS: Final = re.compile( + r'\bON\s+(?:ONLY\s+)?"?\w+"?\s*(?:USING\s+\w+\s*)?\((?P[^)]*)\)', re.IGNORECASE +) +_DROP_INDEX_NAME: Final = re.compile( + r'\bDROP\s+INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE +) +_QUOTED_IDENTIFIER: Final = re.compile(r'"(?P\w+)"') + +_MODEL_BLOCK: Final = re.compile(r"^\s*model\s+(?P\w+)\s*\{(?P.*?)^\s*\}", re.MULTILINE | re.DOTALL) +_ENUM_NAME: Final = re.compile(r"^\s*enum\s+(?P\w+)", re.MULTILINE) +_PRISMA_FIELD: Final = re.compile( + r"^\s*(?P\w+)\s+(?P\w+)(?P\?)?(?P\[\])?(?P[^\n]*)$", re.MULTILINE +) +_FIELD_MAP: Final = re.compile(r'@map\("(?P[^"]+)"\)') +_TABLE_MAP: Final = re.compile(r'@@map\("(?P[^"]+)"\)') _SQL_LITERAL: Final = re.compile(r"(?P\"{3}|'{3})(?P.*?)(?P=quote)", re.DOTALL) _FROM_JOIN: Final = re.compile( @@ -142,6 +196,37 @@ class StarRead: location: str +@dataclass(frozen=True, slots=True) +class Column: + """One column as the base version's Prisma client knows it.""" + + required: bool + prisma_type: str + + +Schema = dict[str, dict[str, Column]] + + +@dataclass(frozen=True, slots=True) +class UpgradeContext: + """Everything a statement is rated against: what this upgrade creates, and what the old pods use.""" + + fresh_tables: frozenset[str] + star_reads: tuple[StarRead, ...] + schema: Schema | None + dropped_indexes: frozenset[str] = frozenset() + + def column(self, table: str, column: str) -> Column | None: + return None if self.schema is None else self.schema.get(table, {}).get(column) + + def knows_table(self, table: str) -> bool: + return self.schema is None or table in self.schema + + def knows_column(self, table: str, column: str) -> bool: + """Whether the old client selects or writes this column. Without a schema, assume it does.""" + return self.schema is None or self.column(table, column) is not None + + @dataclass(frozen=True, slots=True) class Finding: severity: str @@ -167,6 +252,7 @@ class Report: migrations: tuple[str, ...] findings: tuple[Finding, ...] star_reads: tuple[StarRead, ...] + schema_tables: int | None = None @property def worst(self) -> str | None: @@ -249,6 +335,59 @@ def tables_created_before(repo: Path, ref: str) -> frozenset[str]: return frozenset(match.group("table") for match in _CREATE_TABLE.finditer(listing or "")) +def parse_prisma_schema(text: str) -> Schema: + """The tables and columns a Prisma client generated from this schema selects and writes. + + Relation fields are not columns and are skipped; `@map` / `@@map` give the real names. + """ + enums: Final = frozenset(_ENUM_NAME.findall(text)) + schema: Final[Schema] = {} + for block in _MODEL_BLOCK.finditer(text): + body: Final = block.group("body") + table_map: Final = _TABLE_MAP.search(body) + table: Final = table_map.group("name") if table_map is not None else block.group("model") + columns: Final[dict[str, Column]] = {} + for field in _PRISMA_FIELD.finditer(body): + prisma_type: Final = field.group("type") + if prisma_type not in PRISMA_SCALARS and prisma_type not in enums: + continue + field_map: Final = _FIELD_MAP.search(field.group("rest")) + name: Final = field_map.group("name") if field_map is not None else field.group("name") + columns[name] = Column(required=field.group("optional") is None, prisma_type=prisma_type) + schema[table] = columns + return schema + + +def load_base_schema(repo: Path, ref: str) -> Schema | None: + for path in SCHEMA_PATHS: + text: Final = _git_or_none(repo, "show", f"{ref}:{path}") + if text is not None: + parsed: Final = parse_prisma_schema(text) + if parsed: + return parsed + return None + + +def _identifiers(fragment: str) -> tuple[str, ...]: + return tuple(dict.fromkeys(match.group("name") for match in _QUOTED_IDENTIFIER.finditer(fragment))) + + +def _constraint_columns(statement: str) -> tuple[str, ...]: + """The columns of this table that a new constraint checks on every write.""" + listed: Final = [ + name for match in _CONSTRAINT_COLUMNS.finditer(statement) for name in _identifiers(match.group("columns")) + ] + check: Final = _CHECK_BODY.search(statement) + if check is not None: + listed.extend(_identifiers(check.group("body"))) + return tuple(dict.fromkeys(listed)) + + +def _index_columns(statement: str) -> tuple[str, ...]: + match: Final = _INDEX_COLUMNS.search(statement) + return _identifiers(match.group("columns")) if match is not None else () + + def _alias_map(body: str) -> tuple[dict[str, str], tuple[str, ...]]: aliases: Final[dict[str, str]] = {} tables: Final[list[str]] = [] @@ -321,77 +460,204 @@ def _joined_read_for(table: str, star_reads: Sequence[StarRead]) -> StarRead | N return next((read for read in star_reads if table in read.joined), None) -def classify( - migration: str, - statement: str, - fresh_tables: frozenset[str], - star_reads: Sequence[StarRead], -) -> Finding: - """Rate one statement by what it does to a pod still running the base version.""" - table: Final = statement_table(statement) +@dataclass(frozen=True, slots=True) +class _Site: + """One statement under rating, with what the rules need to know about it.""" - def finding(severity: str, effect: str) -> Finding: - return Finding(severity=severity, migration=migration, table=table, statement=statement, effect=effect) + migration: str + statement: str + table: str + context: UpgradeContext + star: StarRead | None - if _CREATE_TABLE.search(statement) is not None: - return finding(INFO, "New table. The previous version does not read or write it.") - if table and table in fresh_tables: - return finding(INFO, "Table is created by this same upgrade, so no old pod reads or writes it yet.") + @property + def where(self) -> str: + return f" (`{self.star.location}`)" if self.star is not None else "" - star: Final = _star_read_for(table, star_reads) - where: Final = f" (`{star.location}`)" if star is not None else "" + def rate(self, severity: str, effect: str) -> Finding: + return Finding( + severity=severity, migration=self.migration, table=self.table, statement=self.statement, effect=effect + ) - if _DROP_COLUMN.search(statement) is not None or _DROP_TABLE.search(statement) is not None: - return finding( + def knows(self, column: str) -> bool: + return self.context.knows_column(self.table, column) + + +def _rule_new_table(site: _Site) -> Finding | None: + if _CREATE_TABLE.search(site.statement) is not None: + return site.rate(INFO, "New table. The previous version does not read or write it.") + if site.table and site.table in site.context.fresh_tables: + return site.rate(INFO, "Table is created by this same upgrade, so no old pod reads or writes it yet.") + return None + + +def _rule_drop(site: _Site) -> Finding | None: + if _DROP_TABLE.search(site.statement) is not None: + if site.context.knows_table(site.table): + return site.rate( + BREAKING, "Drops a table the previous version still queries. Old pods keep failing until they are gone." + ) + return site.rate(INFO, "Drops a table the previous version does not know.") + dropped: Final = _DROP_COLUMN_NAME.search(site.statement) + if dropped is None: + return None + column: Final = dropped.group("column") + if site.knows(column): + return site.rate( BREAKING, - f"Removes something the previous version still reads{where}. Old pods keep failing until they are gone; " - "reconnecting does not help.", + f"Drops `{column}`, which the previous version's client selects on every read of this table{site.where}. " + "Old pods keep failing until they are gone; reconnecting does not help.", ) - if _ALTER_TABLE.search(statement) is not None and _RENAME.search(statement) is not None: - return finding( + return site.rate(INFO, f"Drops `{column}`, a column the previous version does not know.") + + +def _rule_rename(site: _Site) -> Finding | None: + renamed: Final = _RENAME_COLUMN.search(site.statement) + if renamed is not None: + old_name: Final = renamed.group("column") + if site.knows(old_name): + return site.rate( + BREAKING, + f"Renames `{old_name}`, which the previous version still selects and writes by that name{site.where}. " + "Old pods cannot recover on their own.", + ) + return site.rate(INFO, f"Renames `{old_name}`, a column the previous version does not know.") + if _ALTER_TABLE.search(site.statement) is None or _RENAME_TABLE.search(site.statement) is None: + return None + if site.context.knows_table(site.table): + return site.rate(BREAKING, "Renames a table the previous version still queries by its old name.") + return site.rate(INFO, "Renames a table the previous version does not know.") + + +def _rule_not_null(site: _Site) -> Finding | None: + required: Final = [match.group("column") for match in _SET_NOT_NULL_COLUMN.finditer(site.statement)] + if required: + unguarded: Final = [ + column + for column in required + if (known := site.context.column(site.table, column)) is None or not known.required + ] + if unguarded or site.context.schema is None: + return site.rate( + BREAKING, + f"Requires `{'`, `'.join(unguarded or required)}`, which the previous version may write as NULL. " + "Those writes are now rejected.", + ) + return site.rate(INFO, "The previous version already always writes this column, so nothing changes for it.") + if _ADD_COLUMN.search(site.statement) is not None and _adds_not_null_without_default(site.statement): + return site.rate(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") + return None + + +def _rule_type_change(site: _Site) -> Finding | None: + changes: Final = list(_TYPE_CHANGE.finditer(site.statement)) + if not changes: + return None + narrowing: Final = [ + f"`{match.group('column')}` to {match.group('type').upper()}" + for match in changes + if (known := site.context.column(site.table, match.group("column"))) is None + or match.group("type").upper() not in WIDENING.get(known.prisma_type, frozenset()) + ] + if narrowing: + return site.rate( BREAKING, - f"Renames something the previous version still addresses by its old name{where}. Old pods cannot " - "recover on their own.", + f"Changes {', '.join(narrowing)}, which the previous version's client cannot be shown to read. Its " + "prepared plans are rejected once, and the retry may still fail to map the value.", ) - if _SET_NOT_NULL.search(statement) is not None: - return finding(BREAKING, "Rows the previous version writes without this column are now rejected.") - if _ADD_COLUMN.search(statement) is not None and _adds_not_null_without_default(statement): - return finding(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") - if _ALTER_TYPE.search(statement) is not None: - return finding( - PREPARED_PLAN, - "Changes a column's type, so every prepared query on the previous version that returns it is rejected " - "with `cached plan must not change result type` until its connection is recreated. If the new type " - "is one the old client cannot map, it keeps failing after that.", + return site.rate( + PREPARED_PLAN, + "Widens a column's type. Every prepared query on the previous version that returns it is rejected with " + "`cached plan must not change result type` until its connection is recreated, then reads it fine.", + ) + + +def _rule_constraint(site: _Site) -> Finding | None: + if _ADD_CONSTRAINT.search(site.statement) is not None: + checked: Final = [column for column in _constraint_columns(site.statement) if site.knows(column)] + if checked: + return site.rate( + WRITE_REJECT, + f"Adds a rule on `{'`, `'.join(checked)}` that the previous version never enforced. Rows it writes " + "that violate the rule are now rejected; whether any do depends on the data.", + ) + if _NOT_VALID.search(site.statement) is not None: + return site.rate( + INFO, "Constraint on columns the previous version does not write, with no validation scan." + ) + return site.rate(LOCK, "Validates the constraint against every existing row while holding the table lock.") + if _UNIQUE_INDEX.search(site.statement) is None: + return None + covered: Final = _index_columns(site.statement) + replaces: Final = any(name.startswith(f"{site.table}_") for name in site.context.dropped_indexes) + if covered and all(site.knows(column) for column in covered) and not replaces: + return site.rate( + WRITE_REJECT, + f"Makes `{'`, `'.join(covered)}` unique, which the previous version never enforced. Duplicate rows it " + "writes are now rejected; whether any are depends on the data.", ) - if _ADD_COLUMN.search(statement) is not None and star is not None: - return finding( - PREPARED_PLAN, - f"Read whole-row by `{_shape(star)}`{where}, so the prepared plans cached on the previous version's " - "pooled connections are rejected with `cached plan must not change result type` until those " - "connections are recreated.", - ) - if _CREATE_INDEX.search(statement) is not None and _CONCURRENTLY.search(statement) is None: - joined: Final = _joined_read_for(table, star_reads) + return None + + +def _rule_whole_row_read(site: _Site) -> Finding | None: + if _ADD_COLUMN.search(site.statement) is None or site.star is None: + return None + return site.rate( + PREPARED_PLAN, + f"Read whole-row by `{_shape(site.star)}`{site.where}, so the prepared plans cached on the previous " + "version's pooled connections are rejected with `cached plan must not change result type` until those " + "connections are recreated.", + ) + + +def _rule_lock(site: _Site) -> Finding | None: + if _CREATE_INDEX.search(site.statement) is not None and _CONCURRENTLY.search(site.statement) is None: + joined: Final = _joined_read_for(site.table, site.context.star_reads) read_note: Final = ( f" The table is joined by `{_shape(joined)}` (`{joined.location}`), so that read waits too." if joined is not None else "" ) - return finding( + return site.rate( LOCK, "Blocks writes to the table (reads continue) until the index is built; the wait scales with the " f"table's size.{read_note}", ) - if _ADD_CONSTRAINT.search(statement) is not None and _NOT_VALID.search(statement) is None: - return finding(LOCK, "Validates the constraint against every existing row while holding the table lock.") - if _UPDATE.search(statement) is not None: - return finding(LOCK, "Rewrites existing rows, so it holds row locks for as long as the backfill runs.") - if _ADD_COLUMN.search(statement) is not None: - return finding(INFO, "Adds a nullable column to a table the previous version does not read whole-row.") - if _INSERT.search(statement) is not None: - return finding(INFO, "Inserts rows. The previous version is not affected.") - return finding(INFO, "No effect on the previous version.") + if _UPDATE.search(site.statement) is not None: + return site.rate(LOCK, "Rewrites existing rows, so it holds row locks for as long as the backfill runs.") + return None + + +def _rule_rest(site: _Site) -> Finding: + if _ADD_COLUMN.search(site.statement) is not None: + return site.rate(INFO, "Adds a nullable column to a table the previous version does not read whole-row.") + if _INSERT.search(site.statement) is not None: + return site.rate(INFO, "Inserts rows. The previous version is not affected.") + return site.rate(INFO, "No effect on the previous version.") + + +# Worst-first: the first rule that has something to say about a statement decides its rating. +RULES: Final = ( + _rule_new_table, + _rule_drop, + _rule_rename, + _rule_not_null, + _rule_type_change, + _rule_constraint, + _rule_whole_row_read, + _rule_lock, +) + + +def classify(migration: str, statement: str, context: UpgradeContext) -> Finding: + """Rate one statement by what it does to a pod still running the base version.""" + table: Final = statement_table(statement) + site: Final = _Site(migration, statement, table, context, _star_read_for(table, context.star_reads)) + for rule in RULES: + found: Final = rule(site) + if found is not None: + return found + return _rule_rest(site) def _adds_not_null_without_default(statement: str) -> bool: @@ -419,14 +685,26 @@ def build_report(repo: Path, base: str, head: str) -> Report: ) # A re-declared `CREATE TABLE IF NOT EXISTS` of a table the old pods already use is not a new table. fresh_tables: Final = created_here - tables_created_before(repo, base) + schema: Final = load_base_schema(repo, base) findings: Final[list[Finding]] = [] for name, sql in sql_by_migration.items(): - for statement in split_statements(sql): - findings.append(classify(name, statement, fresh_tables, star_reads)) + statements: Final = split_statements(sql) + dropped: Final = frozenset( + match.group("index") for statement in statements if (match := _DROP_INDEX_NAME.search(statement)) + ) + context: Final = UpgradeContext(fresh_tables, star_reads, schema, dropped) + findings.extend(classify(name, statement, context) for statement in statements) ranked: Final = sorted(findings, key=lambda finding: (SEVERITY_ORDER.index(finding.severity), finding.migration)) - return Report(base=base, head=head, migrations=new_names, findings=tuple(ranked), star_reads=star_reads) + return Report( + base=base, + head=head, + migrations=new_names, + findings=tuple(ranked), + star_reads=star_reads, + schema_tables=None if schema is None else len(schema), + ) PROCEDURE: Final = { @@ -434,6 +712,10 @@ PROCEDURE: Final = { "**Do not run the old and new versions side by side.** Take the previous version out of service before the " "new version applies these migrations, or split the change across two releases." ), + WRITE_REJECT: ( + "Writes from the previous version that break the new rule are rejected until those pods are gone. Confirm " + "the previous version already satisfies it on your data; if you cannot, do not overlap the two versions." + ), PREPARED_PLAN: ( "Expect the queries cited above to fail on pods still running the previous version until their pooled " "connections are recreated. LiteLLM recreates them on the first such failure as of `1.98.0`; on older " @@ -472,7 +754,7 @@ def render_markdown(report: Report) -> str: notable: Final = [finding for finding in report.findings if finding.severity != INFO] for finding in notable[:MAX_ROWS]: lines.append( - f"| {SEVERITY_ICON[finding.severity]} | `{finding.migration}` | `{_clip(finding.statement)}` | " + f"| **{SEVERITY_LABEL[finding.severity]}** | `{finding.migration}` | `{_clip(finding.statement)}` | " f"{finding.effect} |" ) if len(notable) > MAX_ROWS: @@ -483,12 +765,23 @@ def render_markdown(report: Report) -> str: worst: Final = report.worst if worst is None or worst == INFO: - lines.append("| ℹ️ | — | — | Nothing in this release affects a pod running the previous version. |") + lines.append("| **INFO** | - | - | Nothing in this release affects a pod running the previous version. |") lines.extend(["", PROCEDURE[worst or INFO], ""]) + evidence: Final[list[str]] = [] + if report.schema_tables is None: + evidence.append( + f"no `schema.prisma` at `{report.base}`, so drops, renames and constraints are rated as if the previous " + "version used every column" + ) + else: + evidence.append( + f"columns the previous version uses: `schema.prisma` at `{report.base}` ({report.schema_tables} tables)" + ) cited: Final = _cited_reads(report) if cited: shapes: Final = ", ".join(sorted({f"`{_shape(read)}` on `{read.table}` (`{read.location}`)" for read in cited})) - lines.append(f"Whole-row reads considered: {shapes}.") + evidence.append(f"whole-row reads: {shapes}") + lines.append(f"Evidence: {'; '.join(evidence)}.") lines.append("") return "\n".join(lines) @@ -513,6 +806,7 @@ def render_json(report: Report) -> str: "base": report.base, "head": report.head, "worst_severity": report.worst, + "schema_tables": report.schema_tables, "migrations": list(report.migrations), "findings": [finding.as_dict() for finding in report.findings], "star_reads": [ @@ -555,7 +849,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--output", help="write the report here instead of stdout") parser.add_argument( "--fail-on", - choices=(BREAKING, PREPARED_PLAN, LOCK), + choices=(BREAKING, WRITE_REJECT, PREPARED_PLAN, LOCK), help="exit 1 when a finding at this severity or worse is present", ) return parser.parse_args(argv) diff --git a/litellm-proxy-extras/migration_runbook.md b/litellm-proxy-extras/migration_runbook.md index 39cad08b303..788b9fc4428 100644 --- a/litellm-proxy-extras/migration_runbook.md +++ b/litellm-proxy-extras/migration_runbook.md @@ -130,13 +130,16 @@ migration set between two refs and rates every statement by what it does to thos | Severity | Meaning | |---|---| -| `breaking` | The old pods cannot recover on their own — a column they read is gone, renamed or retyped, or a constraint now rejects the rows they write | -| `prepared-plan` | The change alters the result type of a query the old pods have prepared - a `SELECT .*` gaining a column, or a column changing type - so the plans cached on their pooled connections are rejected with `cached plan must not change result type` until those connections are recreated ([#36418](https://github.com/BerriAI/litellm/issues/36418)) | +| `breaking` | The old pods cannot recover on their own. A column their client selects is dropped, renamed or changed to a type it cannot read, or a NOT NULL now rejects the rows they write | +| `write-reject` | A new constraint (foreign key, check, unique) on columns the old pods write. Rows they write that violate it are rejected; whether any do depends on the data | +| `prepared-plan` | The result type of a query the old pods have prepared changes: a whole-row `SELECT .*` gains a column, or a column widens. The plans cached on their pooled connections are rejected with `cached plan must not change result type` until those connections are recreated ([#36418](https://github.com/BerriAI/litellm/issues/36418)) | | `lock` | The migration takes a blocking lock, or rewrites rows, on a table already serving traffic | -| `info` | No effect on the old pods (new tables, and changes to them) | +| `info` | No effect on the old pods: new tables, changes to them, and columns the old pods do not know | -The tables read whole-row are discovered from the source tree at `--base` - the code the old pods -are running - not hardcoded, so the report follows those queries as they move. +The evidence is the base version itself, not a hardcoded list. `schema.prisma` at `--base` says +which columns the old Prisma client selects and writes, so dropping a column that schema never +declared is `info`, and dropping one it does declare is `breaking`. The source tree at `--base` +holds the raw `SELECT .*` queries whose plans go stale. The report's footnote names both. `create-release.yml` runs this and puts the result at the top of the release body. To see it for any pair of versions yourself: @@ -147,7 +150,8 @@ python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 --format json ``` Standard library only, so it needs no `uv sync`. `--fail-on breaking` exits non-zero when a -migration in the range is not safe to apply under a rolling upgrade. +migration in the range is not safe to apply under a rolling upgrade; `--fail-on write-reject` +also stops on constraints the old pods might violate. --- diff --git a/tests/test_litellm/test_migration_impact.py b/tests/test_litellm/test_migration_impact.py index c23cd3627e7..7fa115e7318 100644 --- a/tests/test_litellm/test_migration_impact.py +++ b/tests/test_litellm/test_migration_impact.py @@ -72,8 +72,64 @@ def _auth_read() -> object: return reads[0] -def _classify(statement: str, fresh: frozenset[str] = frozenset(), reads: tuple[object, ...] = ()) -> object: - return impact.classify("20260713230852_add_key_type", statement, fresh, reads) +BASE_SCHEMA: Final = """ +enum KeyType { + llm_api + management +} + +model LiteLLM_VerificationToken { + token String @id + key_name String? + spend Float @default(0.0) + models String[] + team_id String? + key_type KeyType? + litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) + budget_id String? + @@index([team_id]) +} + +model LiteLLM_BudgetTable { + budget_id String @id + max_budget Float? + keys LiteLLM_VerificationToken[] +} + +model LiteLLM_ModelTable { + id Int @id @default(autoincrement()) + model_aliases Json? @map("aliases") + created_at DateTime @default(now()) @map("created_at") +} + +model ClaudeCodePlugin { + id String @id + @@map("LiteLLM_ClaudeCodePluginTable") +} +""" + + +def _context( + fresh: frozenset[str] = frozenset(), + reads: tuple[object, ...] = (), + schema: object = None, + dropped: frozenset[str] = frozenset(), +) -> object: + return impact.UpgradeContext(fresh, reads, schema, dropped) + + +def _classify( + statement: str, + fresh: frozenset[str] = frozenset(), + reads: tuple[object, ...] = (), + schema: object = None, + dropped: frozenset[str] = frozenset(), +) -> object: + return impact.classify("20260713230852_add_key_type", statement, _context(fresh, reads, schema, dropped)) + + +def _base_schema() -> object: + return impact.parse_prisma_schema(BASE_SCHEMA) def test_split_statements_descends_into_dollar_quoted_blocks() -> None: @@ -146,10 +202,29 @@ def test_index_builds_are_rated_by_whether_they_lock() -> None: assert concurrent.severity == impact.INFO -def test_unvalidated_constraint_is_not_a_lock() -> None: - validating = _classify('ALTER TABLE "LiteLLM_SpendLogs" ADD CONSTRAINT "fk" FOREIGN KEY ("t") REFERENCES "T"("id")') +def test_constraint_on_a_column_the_old_client_writes_can_reject_its_writes() -> None: + # NOT VALID only skips the scan of existing rows; new writes from the old pods are still checked. + schema = _base_schema() + checked = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "fk" FOREIGN KEY ("budget_id") ' + 'REFERENCES "LiteLLM_BudgetTable"("budget_id") NOT VALID', + schema=schema, + ) + assert checked.severity == impact.WRITE_REJECT + assert "`budget_id`" in checked.effect + + +def test_constraint_on_a_column_the_old_client_does_not_know_is_rated_by_its_scan() -> None: + schema = _base_schema() + validating = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "fk" FOREIGN KEY ("project_id") ' + 'REFERENCES "LiteLLM_ProjectTable"("project_id")', + schema=schema, + ) deferred = _classify( - 'ALTER TABLE "LiteLLM_SpendLogs" ADD CONSTRAINT "fk" FOREIGN KEY ("t") REFERENCES "T"("id") NOT VALID' + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "fk" FOREIGN KEY ("project_id") ' + 'REFERENCES "LiteLLM_ProjectTable"("project_id") NOT VALID', + schema=schema, ) assert validating.severity == impact.LOCK assert deferred.severity == impact.INFO @@ -180,7 +255,7 @@ def test_markdown_leads_with_the_worst_finding_and_its_procedure() -> None: finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) rendered = impact.render_markdown(_report(findings=(finding,))) assert "20260713230852_add_key_type" in rendered - assert impact.SEVERITY_ICON[impact.PREPARED_PLAN] in rendered + assert "| **PREPARED-PLAN** |" in rendered assert impact.PROCEDURE[impact.PREPARED_PLAN] in rendered assert "`SELECT v.*` on `LiteLLM_VerificationToken`" in rendered @@ -188,7 +263,7 @@ def test_markdown_leads_with_the_worst_finding_and_its_procedure() -> None: def test_markdown_caps_the_table_and_says_where_the_rest_is() -> None: finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) rendered = impact.render_markdown(_report(findings=tuple([finding] * (impact.MAX_ROWS + 3)))) - assert rendered.count("| ⚠️ |") == impact.MAX_ROWS + assert rendered.count("| **PREPARED-PLAN** |") == impact.MAX_ROWS assert "…and 3 more" in rendered @@ -247,7 +322,8 @@ def test_foreign_key_on_update_is_not_a_backfill() -> None: 'ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "fk" FOREIGN KEY ("token") ' 'REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE CASCADE ON UPDATE CASCADE NOT VALID' ) - assert finding.severity == impact.INFO + assert finding.severity == impact.WRITE_REJECT + assert "backfill" not in finding.effect backfill = _classify('UPDATE "LiteLLM_SpendLogs" SET "created_at" = "startTime" WHERE "created_at" IS NULL') assert backfill.severity == impact.LOCK @@ -257,12 +333,32 @@ def test_index_rename_is_not_breaking() -> None: assert finding.severity == impact.INFO -def test_type_change_is_a_prepared_plan_change_not_a_permanent_break() -> None: - finding = _classify('ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT') +SPEND_SCHEMA: Final = "model LiteLLM_DailyTagSpend {\n id String @id\n prompt_tokens Int\n}\n" + + +def test_widening_a_type_is_a_prepared_plan_change() -> None: + finding = _classify( + 'ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT', + schema=impact.parse_prisma_schema(SPEND_SCHEMA), + ) assert finding.severity == impact.PREPARED_PLAN assert "cached plan must not change result type" in finding.effect +def test_a_type_the_old_client_cannot_read_is_breaking() -> None: + finding = _classify( + 'ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE TEXT', + schema=impact.parse_prisma_schema(SPEND_SCHEMA), + ) + assert finding.severity == impact.BREAKING + assert "`prompt_tokens` to TEXT" in finding.effect + + +def test_type_change_without_a_schema_to_check_against_is_breaking() -> None: + finding = _classify('ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT') + assert finding.severity == impact.BREAKING + + def test_not_null_is_judged_per_add_column_clause() -> None: one_default_is_not_enough = _classify( 'ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "a" TEXT NOT NULL DEFAULT \'x\', ADD COLUMN "b" TEXT NOT NULL' @@ -325,6 +421,7 @@ def _two_version_repo(tmp_path: Path) -> Path: _git(repo, "config", "user.name", "test") (repo / "litellm" / "proxy").mkdir(parents=True) (repo / "litellm" / "proxy" / "utils.py").write_text(BASE_SOURCE, encoding="utf-8") + (repo / "schema.prisma").write_text(BASE_SCHEMA, encoding="utf-8") _write_migration( repo, "20250101000000_baseline", @@ -344,7 +441,11 @@ def _two_version_repo(tmp_path: Path) -> Path: 'CREATE TABLE IF NOT EXISTS "LiteLLM_VerificationToken" ("token" TEXT PRIMARY KEY);\n' 'ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT;\n' 'CREATE TABLE "LiteLLM_NewTable" ("id" TEXT PRIMARY KEY);\n' - 'CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id");\n', + 'CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id");\n' + # A column the base schema never declared: the old client cannot miss it. + 'ALTER TABLE "LiteLLM_TeamTable" DROP COLUMN IF EXISTS "legacy_flag";\n' + # A column it does declare: the old client selects it on every read. + 'ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name";\n', ) _git(repo, "add", ".") _git(repo, "commit", "-q", "-m", "head") @@ -366,7 +467,12 @@ def test_report_rates_migrations_against_the_queries_the_old_pods_run(tmp_path: assert add_column.severity == impact.PREPARED_PLAN new_index = by_statement['CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id")'] assert new_index.severity == impact.INFO - assert report.worst == impact.PREPARED_PLAN + unknown_drop = by_statement['ALTER TABLE "LiteLLM_TeamTable" DROP COLUMN IF EXISTS "legacy_flag"'] + assert unknown_drop.severity == impact.INFO + known_drop = by_statement['ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name"'] + assert known_drop.severity == impact.BREAKING + assert report.schema_tables == 4 + assert report.worst == impact.BREAKING def test_report_for_the_same_ref_twice_has_nothing_to_say(tmp_path: Path) -> None: @@ -375,3 +481,89 @@ def test_report_for_the_same_ref_twice_has_nothing_to_say(tmp_path: Path) -> Non assert report.migrations == () assert report.findings == () assert "No database schema changes since `v1.1.0`." in impact.render_markdown(report) + + +def test_schema_parser_reads_columns_maps_and_enums_and_skips_relations() -> None: + schema = _base_schema() + token = schema["LiteLLM_VerificationToken"] + assert token["token"] == impact.Column(required=True, prisma_type="String") + assert token["key_name"] == impact.Column(required=False, prisma_type="String") + assert token["models"].required # a list column is NOT NULL + assert token["key_type"].prisma_type == "KeyType" # enum-typed column + assert "litellm_budget_table" not in token # relation field, not a column + assert "aliases" in schema["LiteLLM_ModelTable"] # @map renames the column + assert "model_aliases" not in schema["LiteLLM_ModelTable"] + assert "LiteLLM_ClaudeCodePluginTable" in schema # @@map renames the table + assert "ClaudeCodePlugin" not in schema + + +def test_dropping_a_column_the_old_client_never_selects_is_not_breaking() -> None: + schema = _base_schema() + known = _classify('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name"', schema=schema) + unknown = _classify('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "settings_updated_at"', schema=schema) + assert known.severity == impact.BREAKING + assert "`key_name`" in known.effect + assert unknown.severity == impact.INFO + + +def test_renaming_a_column_is_rated_by_whether_the_old_client_uses_it() -> None: + schema = _base_schema() + known = _classify('ALTER TABLE "LiteLLM_VerificationToken" RENAME COLUMN "team_id" TO "owner_team"', schema=schema) + unknown = _classify('ALTER TABLE "LiteLLM_VerificationToken" RENAME COLUMN "tmp" TO "final"', schema=schema) + assert known.severity == impact.BREAKING + assert unknown.severity == impact.INFO + + +def test_set_not_null_is_breaking_only_where_the_old_client_may_write_null() -> None: + schema = _base_schema() + optional_at_base = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ALTER COLUMN "key_name" SET NOT NULL', schema=schema + ) + required_at_base = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ALTER COLUMN "spend" SET NOT NULL', schema=schema + ) + assert optional_at_base.severity == impact.BREAKING + assert required_at_base.severity == impact.INFO + + +def test_dropping_a_table_the_old_client_does_not_know_is_informational() -> None: + schema = _base_schema() + known = _classify('DROP TABLE "LiteLLM_BudgetTable"', schema=schema) + unknown = _classify('DROP TABLE "LiteLLM_TmpTable"', schema=schema) + assert known.severity == impact.BREAKING + assert unknown.severity == impact.INFO + + +def test_unique_index_on_columns_the_old_client_writes_can_reject_duplicates() -> None: + finding = _classify( + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_key" ON "LiteLLM_VerificationToken"("key_name")', + schema=_base_schema(), + ) + assert finding.severity == impact.WRITE_REJECT + + +def test_unique_index_that_replaces_a_dropped_one_is_only_a_lock() -> None: + # The Daily*Spend migrations drop the old unique key and create a wider one in the same file. + finding = _classify( + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_team_id_key" ON "LiteLLM_VerificationToken"' + '("key_name", "team_id")', + schema=_base_schema(), + dropped=frozenset({"LiteLLM_VerificationToken_key_name_key"}), + ) + assert finding.severity == impact.LOCK + + +def test_unique_index_on_a_new_column_is_only_a_lock() -> None: + finding = _classify( + 'CREATE UNIQUE INDEX "idx" ON "LiteLLM_VerificationToken"("settings_updated_at")', schema=_base_schema() + ) + assert finding.severity == impact.LOCK + + +def test_markdown_names_the_schema_it_used_as_evidence() -> None: + report = impact.Report( + base="v1.93.0", head="v1.99.0", migrations=("m",), findings=(), star_reads=(), schema_tables=65 + ) + assert "`schema.prisma` at `v1.93.0` (65 tables)" in impact.render_markdown(report) + blind = impact.Report(base="v1.93.0", head="v1.99.0", migrations=("m",), findings=(), star_reads=()) + assert "no `schema.prisma` at `v1.93.0`" in impact.render_markdown(blind) From fb9fcb1615aa4e3992dfb0a96a7e50223e8ddf9f Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Fri, 18 Sep 2026 12:55:50 +0900 Subject: [PATCH 3/6] fix(ci): read bare constraint columns and judge unique keys by what they replace Second review round on #41744. A CHECK written without quotes named no columns, so it rated info; and a unique index was exempted as a replacement on the strength of its name prefix, which would also have exempted a narrower key. Names in a constraint are now read quoted or bare and kept only when the base schema declares them for that table, so `CHECK (status = 'active')` finds `status` and ignores the literal. Without a base schema every new constraint is write-reject. A column this upgrade adds with a DEFAULT counts as written by the old pods (they insert the default), so a composite unique index over it still binds them. A unique index is exempt only when a unique index dropped in the same migration existed at the base on the same table with a subset of its columns: the new key then rejects nothing the old one did not. The base's unique indexes come from its migrations. The jwt_issuer migration cited in review drops (jwt_claim_name, jwt_claim_value) and recreates (jwt_issuer, jwt_claim_name, jwt_claim_value), so it stays a lock on evidence rather than on a name prefix. Co-Authored-By: Claude Fable 5.1 --- ci_cd/migration_impact.py | 103 ++++++++++++++++---- tests/test_litellm/test_migration_impact.py | 83 ++++++++++++++-- 2 files changed, 160 insertions(+), 26 deletions(-) diff --git a/ci_cd/migration_impact.py b/ci_cd/migration_impact.py index 9211e3314f5..6b2d75cded8 100755 --- a/ci_cd/migration_impact.py +++ b/ci_cd/migration_impact.py @@ -41,7 +41,7 @@ import re import subprocess import sys from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Final @@ -167,6 +167,11 @@ _DROP_INDEX_NAME: Final = re.compile( r'\bDROP\s+INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE ) _QUOTED_IDENTIFIER: Final = re.compile(r'"(?P\w+)"') +_BARE_WORD: Final = re.compile(r"\b(?P[A-Za-z_]\w*)\b") +_ADDED_COLUMN_NAME: Final = re.compile(r'\bADD\s+COLUMN\s+(?:IF\s+NOT\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE) +_INDEX_NAME: Final = re.compile( + r'\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE +) _MODEL_BLOCK: Final = re.compile(r"^\s*model\s+(?P\w+)\s*\{(?P.*?)^\s*\}", re.MULTILINE | re.DOTALL) _ENUM_NAME: Final = re.compile(r"^\s*enum\s+(?P\w+)", re.MULTILINE) @@ -215,6 +220,10 @@ class UpgradeContext: star_reads: tuple[StarRead, ...] schema: Schema | None dropped_indexes: frozenset[str] = frozenset() + # (table, column) added by this upgrade with a DEFAULT: the old pods write that default on every insert. + defaulted_columns: frozenset[tuple[str, str]] = frozenset() + # Unique indexes that exist at the base, by name: what a dropped-and-recreated key used to cover. + base_unique_indexes: dict[str, tuple[str, tuple[str, ...]]] = field(default_factory=dict) def column(self, table: str, column: str) -> Column | None: return None if self.schema is None else self.schema.get(table, {}).get(column) @@ -226,6 +235,19 @@ class UpgradeContext: """Whether the old client selects or writes this column. Without a schema, assume it does.""" return self.schema is None or self.column(table, column) is not None + def writes(self, table: str, column: str) -> bool: + """Whether rows the old pods insert carry a value here: a column they know, or a default they get.""" + return self.knows_column(table, column) or (table, column) in self.defaulted_columns + + def replaced_key(self, table: str, columns: Sequence[str]) -> bool: + """Whether a dropped unique index on this table covered a subset of `columns`: the new key then rejects + nothing the old key did not already reject.""" + for name in self.dropped_indexes: + found: Final = self.base_unique_indexes.get(name) + if found is not None and found[0] == table and set(found[1]) <= set(columns): + return True + return False + @dataclass(frozen=True, slots=True) class Finding: @@ -347,13 +369,13 @@ def parse_prisma_schema(text: str) -> Schema: table_map: Final = _TABLE_MAP.search(body) table: Final = table_map.group("name") if table_map is not None else block.group("model") columns: Final[dict[str, Column]] = {} - for field in _PRISMA_FIELD.finditer(body): - prisma_type: Final = field.group("type") + for declared in _PRISMA_FIELD.finditer(body): + prisma_type: Final = declared.group("type") if prisma_type not in PRISMA_SCALARS and prisma_type not in enums: continue - field_map: Final = _FIELD_MAP.search(field.group("rest")) - name: Final = field_map.group("name") if field_map is not None else field.group("name") - columns[name] = Column(required=field.group("optional") is None, prisma_type=prisma_type) + field_map: Final = _FIELD_MAP.search(declared.group("rest")) + name: Final = field_map.group("name") if field_map is not None else declared.group("name") + columns[name] = Column(required=declared.group("optional") is None, prisma_type=prisma_type) schema[table] = columns return schema @@ -369,11 +391,41 @@ def load_base_schema(repo: Path, ref: str) -> Schema | None: def _identifiers(fragment: str) -> tuple[str, ...]: - return tuple(dict.fromkeys(match.group("name") for match in _QUOTED_IDENTIFIER.finditer(fragment))) + """Every name in the fragment, quoted or bare. Bare words include keywords and literals, so callers keep + only the ones that are columns of the table at hand.""" + quoted: Final = [match.group("name") for match in _QUOTED_IDENTIFIER.finditer(fragment)] + unquoted: Final = _QUOTED_IDENTIFIER.sub(" ", re.sub(r"'[^']*'", " ", fragment)) + bare: Final = [match.group("name") for match in _BARE_WORD.finditer(unquoted)] + return tuple(dict.fromkeys(quoted + bare)) + + +def _defaulted_columns(statements: Sequence[str]) -> frozenset[tuple[str, str]]: + """(table, column) pairs these statements add with a DEFAULT.""" + found: Final[set[tuple[str, str]]] = set() + for statement in statements: + table: Final = statement_table(statement) + for clause in _CLAUSE_SPLIT.split(statement): + added: Final = _ADDED_COLUMN_NAME.search(clause) + if added is not None and _DEFAULT.search(clause) is not None: + found.add((table, added.group("column"))) + return frozenset(found) + + +def unique_indexes_at(repo: Path, ref: str) -> dict[str, tuple[str, tuple[str, ...]]]: + """Every unique index some migration at `ref` creates, by name: its table and columns.""" + listing: Final = _git_or_none( + repo, "grep", "-h", "-i", "--fixed-strings", "CREATE UNIQUE INDEX", ref, "--", f"{MIGRATIONS_DIR}/" + ) + indexes: Final[dict[str, tuple[str, tuple[str, ...]]]] = {} + for line in (listing or "").splitlines(): + named: Final = _INDEX_NAME.search(line) + if named is not None: + indexes[named.group("index")] = (statement_table(line), _index_columns(line)) + return indexes def _constraint_columns(statement: str) -> tuple[str, ...]: - """The columns of this table that a new constraint checks on every write.""" + """Names a new constraint mentions: the listed columns, and every name inside a CHECK expression.""" listed: Final = [ name for match in _CONSTRAINT_COLUMNS.finditer(statement) for name in _identifiers(match.group("columns")) ] @@ -574,7 +626,15 @@ def _rule_type_change(site: _Site) -> Finding | None: def _rule_constraint(site: _Site) -> Finding | None: if _ADD_CONSTRAINT.search(site.statement) is not None: - checked: Final = [column for column in _constraint_columns(site.statement) if site.knows(column)] + if site.context.schema is None: + return site.rate( + WRITE_REJECT, + "Adds a rule the previous version never enforced. Rows it writes that violate the rule are now " + "rejected; whether any do depends on the data.", + ) + checked: Final = [ + column for column in _constraint_columns(site.statement) if site.context.writes(site.table, column) + ] if checked: return site.rate( WRITE_REJECT, @@ -589,14 +649,15 @@ def _rule_constraint(site: _Site) -> Finding | None: if _UNIQUE_INDEX.search(site.statement) is None: return None covered: Final = _index_columns(site.statement) - replaces: Final = any(name.startswith(f"{site.table}_") for name in site.context.dropped_indexes) - if covered and all(site.knows(column) for column in covered) and not replaces: - return site.rate( - WRITE_REJECT, - f"Makes `{'`, `'.join(covered)}` unique, which the previous version never enforced. Duplicate rows it " - "writes are now rejected; whether any are depends on the data.", - ) - return None + if not covered or not all(site.context.writes(site.table, column) for column in covered): + return None + if site.context.replaced_key(site.table, covered): + return None # the key it replaces already rejected everything this one does + return site.rate( + WRITE_REJECT, + f"Makes `{'`, `'.join(covered)}` unique, which the previous version never enforced. Duplicate rows it " + "writes are now rejected; whether any are depends on the data.", + ) def _rule_whole_row_read(site: _Site) -> Finding | None: @@ -686,14 +747,16 @@ def build_report(repo: Path, base: str, head: str) -> Report: # A re-declared `CREATE TABLE IF NOT EXISTS` of a table the old pods already use is not a new table. fresh_tables: Final = created_here - tables_created_before(repo, base) schema: Final = load_base_schema(repo, base) + statements_by_migration: Final = {name: split_statements(sql) for name, sql in sql_by_migration.items()} + defaulted: Final = _defaulted_columns([s for statements in statements_by_migration.values() for s in statements]) + base_unique: Final = unique_indexes_at(repo, base) findings: Final[list[Finding]] = [] - for name, sql in sql_by_migration.items(): - statements: Final = split_statements(sql) + for name, statements in statements_by_migration.items(): dropped: Final = frozenset( match.group("index") for statement in statements if (match := _DROP_INDEX_NAME.search(statement)) ) - context: Final = UpgradeContext(fresh_tables, star_reads, schema, dropped) + context: Final = UpgradeContext(fresh_tables, star_reads, schema, dropped, defaulted, base_unique) findings.extend(classify(name, statement, context) for statement in statements) ranked: Final = sorted(findings, key=lambda finding: (SEVERITY_ORDER.index(finding.severity), finding.migration)) diff --git a/tests/test_litellm/test_migration_impact.py b/tests/test_litellm/test_migration_impact.py index 7fa115e7318..95aff2b8258 100644 --- a/tests/test_litellm/test_migration_impact.py +++ b/tests/test_litellm/test_migration_impact.py @@ -114,8 +114,10 @@ def _context( reads: tuple[object, ...] = (), schema: object = None, dropped: frozenset[str] = frozenset(), + defaulted: frozenset[tuple[str, str]] = frozenset(), + base_unique: dict[str, tuple[str, tuple[str, ...]]] | None = None, ) -> object: - return impact.UpgradeContext(fresh, reads, schema, dropped) + return impact.UpgradeContext(fresh, reads, schema, dropped, defaulted, base_unique or {}) def _classify( @@ -124,8 +126,11 @@ def _classify( reads: tuple[object, ...] = (), schema: object = None, dropped: frozenset[str] = frozenset(), + defaulted: frozenset[tuple[str, str]] = frozenset(), + base_unique: dict[str, tuple[str, tuple[str, ...]]] | None = None, ) -> object: - return impact.classify("20260713230852_add_key_type", statement, _context(fresh, reads, schema, dropped)) + context = _context(fresh, reads, schema, dropped, defaulted, base_unique) + return impact.classify("20260713230852_add_key_type", statement, context) def _base_schema() -> object: @@ -426,7 +431,8 @@ def _two_version_repo(tmp_path: Path) -> Path: repo, "20250101000000_baseline", 'CREATE TABLE "LiteLLM_VerificationToken" ("token" TEXT PRIMARY KEY);\n' - 'CREATE TABLE "LiteLLM_TeamTable" ("team_id" TEXT PRIMARY KEY);\n', + 'CREATE TABLE "LiteLLM_TeamTable" ("team_id" TEXT PRIMARY KEY);\n' + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_key" ON "LiteLLM_VerificationToken"("key_name");\n', ) (repo / impact.MIGRATIONS_DIR / "migration_lock.toml").write_text('provider = "postgresql"\n', encoding="utf-8") _git(repo, "add", ".") @@ -447,6 +453,15 @@ def _two_version_repo(tmp_path: Path) -> Path: # A column it does declare: the old client selects it on every read. 'ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name";\n', ) + _write_migration( + repo, + "20250202000000_scope_key_name_by_issuer", + # The jwt_issuer pattern: drop the old key, add a defaulted column, recreate the key wider. + 'DROP INDEX IF EXISTS "LiteLLM_VerificationToken_key_name_key";\n' + 'ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "issuer" TEXT NOT NULL DEFAULT \'\';\n' + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_issuer_key_name_key" ON "LiteLLM_VerificationToken"' + '("issuer", "key_name");\n', + ) _git(repo, "add", ".") _git(repo, "commit", "-q", "-m", "head") _git(repo, "tag", "v1.1.0") @@ -457,7 +472,7 @@ def test_report_rates_migrations_against_the_queries_the_old_pods_run(tmp_path: repo = _two_version_repo(tmp_path) report = impact.build_report(repo, "v1.0.0", "v1.1.0") - assert report.migrations == ("20250201000000_add_key_type",) + assert report.migrations == ("20250201000000_add_key_type", "20250202000000_scope_key_name_by_issuer") # The whole-row read exists only at v1.0.0; that is the version whose plans go stale. assert [read.table for read in report.star_reads] == ["LiteLLM_VerificationToken"] assert report.star_reads[0].location.startswith("litellm/proxy/utils.py:") @@ -471,6 +486,11 @@ def test_report_rates_migrations_against_the_queries_the_old_pods_run(tmp_path: assert unknown_drop.severity == impact.INFO known_drop = by_statement['ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name"'] assert known_drop.severity == impact.BREAKING + widened_key = by_statement[ + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_issuer_key_name_key" ON "LiteLLM_VerificationToken"' + '("issuer", "key_name")' + ] + assert widened_key.severity == impact.LOCK # widens the key it drops, so nothing new is rejected assert report.schema_tables == 4 assert report.worst == impact.BREAKING @@ -542,17 +562,68 @@ def test_unique_index_on_columns_the_old_client_writes_can_reject_duplicates() - assert finding.severity == impact.WRITE_REJECT -def test_unique_index_that_replaces_a_dropped_one_is_only_a_lock() -> None: - # The Daily*Spend migrations drop the old unique key and create a wider one in the same file. +TOKEN_KEY_NAME_KEY: Final = {"LiteLLM_VerificationToken_key_name_key": ("LiteLLM_VerificationToken", ("key_name",))} + + +def test_unique_index_that_widens_a_dropped_key_is_only_a_lock() -> None: + # The Daily*Spend migrations drop the old unique key and create a wider one in the same file: every + # duplicate the new key rejects, the old key already rejected. finding = _classify( 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_team_id_key" ON "LiteLLM_VerificationToken"' '("key_name", "team_id")', schema=_base_schema(), dropped=frozenset({"LiteLLM_VerificationToken_key_name_key"}), + base_unique=TOKEN_KEY_NAME_KEY, ) assert finding.severity == impact.LOCK +def test_unique_index_that_narrows_a_dropped_key_can_reject_writes() -> None: + # Dropping (key_name, team_id) for (key_name) is stricter: rows that only differed by team now collide. + finding = _classify( + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_key" ON "LiteLLM_VerificationToken"("key_name")', + schema=_base_schema(), + dropped=frozenset({"LiteLLM_VerificationToken_key_name_team_id_key"}), + base_unique={ + "LiteLLM_VerificationToken_key_name_team_id_key": ("LiteLLM_VerificationToken", ("key_name", "team_id")) + }, + ) + assert finding.severity == impact.WRITE_REJECT + + +def test_unique_index_over_a_new_defaulted_column_still_binds_the_old_pods() -> None: + # Old pods insert the default, so the new column does not make their rows distinct. + finding = _classify( + 'CREATE UNIQUE INDEX "idx" ON "LiteLLM_VerificationToken"("key_name", "jwt_issuer")', + schema=_base_schema(), + defaulted=frozenset({("LiteLLM_VerificationToken", "jwt_issuer")}), + ) + assert finding.severity == impact.WRITE_REJECT + + +def test_check_constraint_with_unquoted_columns_is_matched_against_the_schema() -> None: + finding = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "chk" CHECK (key_name <> \'active\') NOT VALID', + schema=_base_schema(), + ) + assert finding.severity == impact.WRITE_REJECT + assert "`key_name`" in finding.effect + assert "active" not in finding.effect # a string literal is not a column + + +def test_constraint_without_a_schema_to_check_against_is_write_reject() -> None: + finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "chk" CHECK (status = \'active\')') + assert finding.severity == impact.WRITE_REJECT + + +def test_defaulted_columns_are_collected_per_clause() -> None: + statements = [ + 'ALTER TABLE "T" ADD COLUMN "a" TEXT NOT NULL DEFAULT \'\', ADD COLUMN "b" TEXT', + 'ALTER TABLE "U" ADD COLUMN IF NOT EXISTS "c" INT DEFAULT 0', + ] + assert impact._defaulted_columns(statements) == frozenset({("T", "a"), ("U", "c")}) + + def test_unique_index_on_a_new_column_is_only_a_lock() -> None: finding = _classify( 'CREATE UNIQUE INDEX "idx" ON "LiteLLM_VerificationToken"("settings_updated_at")', schema=_base_schema() From be7788d47f0eec8d6ae3272daa6f8af65ed0e485 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Fri, 18 Sep 2026 13:03:29 +0900 Subject: [PATCH 4/6] fix(ci): parse base unique indexes per statement and drop the mutable accumulators Third review round on #41744. The base's unique indexes were read line by line from git grep, so an index Prisma had wrapped onto two lines lost its table and columns, and a key that replaced it could not be recognised as a widening. There are four such indexes in the migration history. They are now read per statement from each migration file that creates one. The rest is the repository's no-mutation rule: statements, star reads, schema parsing, constraint columns, the report and the rendered markdown are built in one shot as tuples, frozensets and MappingProxyType instead of being appended to. Co-Authored-By: Claude Fable 5.1 --- ci_cd/migration_impact.py | 403 ++++++++++---------- tests/test_litellm/test_migration_impact.py | 6 +- 2 files changed, 209 insertions(+), 200 deletions(-) diff --git a/ci_cd/migration_impact.py b/ci_cd/migration_impact.py index 6b2d75cded8..d9dddf4217b 100755 --- a/ci_cd/migration_impact.py +++ b/ci_cd/migration_impact.py @@ -40,9 +40,10 @@ import json import re import subprocess import sys -from collections.abc import Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path +from types import MappingProxyType from typing import Final MIGRATIONS_DIR: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations" @@ -209,7 +210,7 @@ class Column: prisma_type: str -Schema = dict[str, dict[str, Column]] +Schema = Mapping[str, Mapping[str, Column]] @dataclass(frozen=True, slots=True) @@ -223,7 +224,7 @@ class UpgradeContext: # (table, column) added by this upgrade with a DEFAULT: the old pods write that default on every insert. defaulted_columns: frozenset[tuple[str, str]] = frozenset() # Unique indexes that exist at the base, by name: what a dropped-and-recreated key used to cover. - base_unique_indexes: dict[str, tuple[str, tuple[str, ...]]] = field(default_factory=dict) + base_unique_indexes: Mapping[str, tuple[str, tuple[str, ...]]] = field(default_factory=lambda: MappingProxyType({})) def column(self, table: str, column: str) -> Column | None: return None if self.schema is None else self.schema.get(table, {}).get(column) @@ -306,20 +307,14 @@ def normalize(statement: str) -> str: return " ".join(statement.split()).strip() -def split_statements(sql: str) -> list[str]: +def split_statements(sql: str) -> tuple[str, ...]: """Split a migration into single-line statements, descending into `DO $$ ... $$` bodies.""" cleaned: Final = _BLOCK_COMMENT.sub(" ", _LINE_COMMENT.sub(" ", sql)) - bodies: Final[list[str]] = [] - - def stash(match: re.Match[str]) -> str: - bodies.append(match.group("body")) - return " " - - outer: Final = _DOLLAR_QUOTED.sub(stash, cleaned) - statements: Final = [normalize(part) for part in outer.split(";")] - for body in bodies: - statements.extend(split_statements(body)) - return [statement for statement in statements if statement] + bodies: Final = tuple(match.group("body") for match in _DOLLAR_QUOTED.finditer(cleaned)) + outer: Final = _DOLLAR_QUOTED.sub(" ", cleaned) + top_level: Final = tuple(normalize(part) for part in outer.split(";")) + nested: Final = tuple(statement for body in bodies for statement in split_statements(body)) + return tuple(statement for statement in top_level + nested if statement) def statement_table(statement: str) -> str: @@ -333,13 +328,9 @@ def statement_table(statement: str) -> str: def migration_names(repo: Path, ref: str) -> tuple[str, ...]: """The migration directories at `ref`, skipping `migration_lock.toml` and any stray file.""" listing: Final = _git(repo, "ls-tree", ref, f"{MIGRATIONS_DIR}/") - names: Final[list[str]] = [] - for line in listing.splitlines(): - # \t - meta, _, path = line.partition("\t") - if meta.split()[1:2] == ["tree"]: - names.append(Path(path).name) - return tuple(sorted(names)) + # \t + entries: Final = tuple(line.partition("\t") for line in listing.splitlines()) + return tuple(sorted(Path(path).name for meta, _, path in entries if meta.split()[1:2] == ["tree"])) def read_migration(repo: Path, ref: str, name: str) -> str | None: @@ -357,82 +348,87 @@ def tables_created_before(repo: Path, ref: str) -> frozenset[str]: return frozenset(match.group("table") for match in _CREATE_TABLE.finditer(listing or "")) +def _model_columns(body: str, enums: frozenset[str]) -> Mapping[str, Column]: + declared: Final = tuple(_PRISMA_FIELD.finditer(body)) + return MappingProxyType( + { + (_FIELD_MAP.search(one.group("rest")) or one).group("name"): Column( + required=one.group("optional") is None, prisma_type=one.group("type") + ) + for one in declared + if one.group("type") in PRISMA_SCALARS or one.group("type") in enums + } + ) + + def parse_prisma_schema(text: str) -> Schema: """The tables and columns a Prisma client generated from this schema selects and writes. Relation fields are not columns and are skipped; `@map` / `@@map` give the real names. """ enums: Final = frozenset(_ENUM_NAME.findall(text)) - schema: Final[Schema] = {} - for block in _MODEL_BLOCK.finditer(text): - body: Final = block.group("body") - table_map: Final = _TABLE_MAP.search(body) - table: Final = table_map.group("name") if table_map is not None else block.group("model") - columns: Final[dict[str, Column]] = {} - for declared in _PRISMA_FIELD.finditer(body): - prisma_type: Final = declared.group("type") - if prisma_type not in PRISMA_SCALARS and prisma_type not in enums: - continue - field_map: Final = _FIELD_MAP.search(declared.group("rest")) - name: Final = field_map.group("name") if field_map is not None else declared.group("name") - columns[name] = Column(required=declared.group("optional") is None, prisma_type=prisma_type) - schema[table] = columns - return schema + return MappingProxyType( + { + (_TABLE_MAP.search(block.group("body")) or block).group( + "name" if _TABLE_MAP.search(block.group("body")) else "model" + ): _model_columns(block.group("body"), enums) + for block in _MODEL_BLOCK.finditer(text) + } + ) def load_base_schema(repo: Path, ref: str) -> Schema | None: - for path in SCHEMA_PATHS: - text: Final = _git_or_none(repo, "show", f"{ref}:{path}") - if text is not None: - parsed: Final = parse_prisma_schema(text) - if parsed: - return parsed - return None + texts: Final = (_git_or_none(repo, "show", f"{ref}:{path}") for path in SCHEMA_PATHS) + return next((parsed for text in texts if text is not None and (parsed := parse_prisma_schema(text))), None) def _identifiers(fragment: str) -> tuple[str, ...]: """Every name in the fragment, quoted or bare. Bare words include keywords and literals, so callers keep only the ones that are columns of the table at hand.""" - quoted: Final = [match.group("name") for match in _QUOTED_IDENTIFIER.finditer(fragment)] + quoted: Final = tuple(match.group("name") for match in _QUOTED_IDENTIFIER.finditer(fragment)) unquoted: Final = _QUOTED_IDENTIFIER.sub(" ", re.sub(r"'[^']*'", " ", fragment)) - bare: Final = [match.group("name") for match in _BARE_WORD.finditer(unquoted)] + bare: Final = tuple(match.group("name") for match in _BARE_WORD.finditer(unquoted)) return tuple(dict.fromkeys(quoted + bare)) -def _defaulted_columns(statements: Sequence[str]) -> frozenset[tuple[str, str]]: +def _defaulted_columns(statements: Iterable[str]) -> frozenset[tuple[str, str]]: """(table, column) pairs these statements add with a DEFAULT.""" - found: Final[set[tuple[str, str]]] = set() - for statement in statements: - table: Final = statement_table(statement) - for clause in _CLAUSE_SPLIT.split(statement): - added: Final = _ADDED_COLUMN_NAME.search(clause) - if added is not None and _DEFAULT.search(clause) is not None: - found.add((table, added.group("column"))) - return frozenset(found) - - -def unique_indexes_at(repo: Path, ref: str) -> dict[str, tuple[str, tuple[str, ...]]]: - """Every unique index some migration at `ref` creates, by name: its table and columns.""" - listing: Final = _git_or_none( - repo, "grep", "-h", "-i", "--fixed-strings", "CREATE UNIQUE INDEX", ref, "--", f"{MIGRATIONS_DIR}/" + return frozenset( + (statement_table(statement), added.group("column")) + for statement in statements + for clause in _CLAUSE_SPLIT.split(statement) + if (added := _ADDED_COLUMN_NAME.search(clause)) is not None and _DEFAULT.search(clause) is not None + ) + + +def unique_indexes_at(repo: Path, ref: str) -> Mapping[str, tuple[str, tuple[str, ...]]]: + """Every unique index some migration at `ref` creates, by name: its table and columns. + + Read statement by statement, not line by line, because Prisma wraps long index definitions. + """ + listing: Final = _git_or_none( + repo, "grep", "-l", "-i", "--fixed-strings", "CREATE UNIQUE INDEX", ref, "--", f"{MIGRATIONS_DIR}/" + ) + paths: Final = tuple(line.split(":", 1)[1] if ":" in line else line for line in (listing or "").splitlines()) + sources: Final = tuple(text for path in paths if (text := _git_or_none(repo, "show", f"{ref}:{path}")) is not None) + return MappingProxyType( + { + named.group("index"): (statement_table(statement), _index_columns(statement)) + for source in sources + for statement in split_statements(source) + if _UNIQUE_INDEX.search(statement) is not None and (named := _INDEX_NAME.search(statement)) is not None + } ) - indexes: Final[dict[str, tuple[str, tuple[str, ...]]]] = {} - for line in (listing or "").splitlines(): - named: Final = _INDEX_NAME.search(line) - if named is not None: - indexes[named.group("index")] = (statement_table(line), _index_columns(line)) - return indexes def _constraint_columns(statement: str) -> tuple[str, ...]: """Names a new constraint mentions: the listed columns, and every name inside a CHECK expression.""" - listed: Final = [ + listed: Final = tuple( name for match in _CONSTRAINT_COLUMNS.finditer(statement) for name in _identifiers(match.group("columns")) - ] + ) check: Final = _CHECK_BODY.search(statement) - if check is not None: - listed.extend(_identifiers(check.group("body"))) - return tuple(dict.fromkeys(listed)) + inside: Final = _identifiers(check.group("body")) if check is not None else () + return tuple(dict.fromkeys(listed + inside)) def _index_columns(statement: str) -> tuple[str, ...]: @@ -440,63 +436,57 @@ def _index_columns(statement: str) -> tuple[str, ...]: return _identifiers(match.group("columns")) if match is not None else () -def _alias_map(body: str) -> tuple[dict[str, str], tuple[str, ...]]: - aliases: Final[dict[str, str]] = {} - tables: Final[list[str]] = [] - for match in _FROM_JOIN.finditer(body): - table: Final = match.group("table") - if table not in tables: - tables.append(table) - alias: Final = match.group("alias") - if alias is not None and alias.lower() not in ALIAS_STOPWORDS: - aliases[alias] = table - return aliases, tuple(tables) +def _alias_map(body: str) -> tuple[Mapping[str, str], tuple[str, ...]]: + matches: Final = tuple(_FROM_JOIN.finditer(body)) + tables: Final = tuple(dict.fromkeys(match.group("table") for match in matches)) + aliases: Final = MappingProxyType( + { + match.group("alias"): match.group("table") + for match in matches + if match.group("alias") is not None and match.group("alias").lower() not in ALIAS_STOPWORDS + } + ) + return aliases, tables -def star_reads_in_source(text: str, path: str) -> list[StarRead]: +def _star_reads_in_literal(body: str, location: str) -> tuple[StarRead, ...]: + aliases, tables = _alias_map(body) + starred: Final = tuple( + (aliases[match.group("alias")], match.group("alias")) + for match in _STAR_ALIAS.finditer(body) + if match.group("alias") in aliases + ) + bare: Final = ((tables[0], "*"),) if not starred and _BARE_STAR.search(body) and len(tables) == 1 else () + return tuple( + StarRead(table=table, alias=alias, joined=tuple(other for other in tables if other != table), location=location) + for table, alias in dict.fromkeys(starred + bare) + ) + + +def star_reads_in_source(text: str, path: str) -> tuple[StarRead, ...]: """Find whole-row reads (`SELECT v.*`, `SELECT *`) in the SQL literals of one source file.""" - found: Final[list[StarRead]] = [] - for literal in _SQL_LITERAL.finditer(text): - raw: Final = literal.group("body") - if TABLE_MARKER not in raw or "SELECT" not in raw.upper(): - continue - if VIEW_MARKER.search(raw) is not None: - continue + return tuple( + read + for literal in _SQL_LITERAL.finditer(text) + if TABLE_MARKER in literal.group("body") + and "SELECT" in literal.group("body").upper() + and VIEW_MARKER.search(literal.group("body")) is None # `-- Added comma to separate b.* columns` is a comment, not a read. - body: Final = _LINE_COMMENT.sub(" ", raw) - aliases, tables = _alias_map(body) - line: Final = text.count("\n", 0, literal.start()) + 1 - location: Final = f"{path}:{line}" - starred: Final[list[tuple[str, str]]] = [ - (aliases[match.group("alias")], match.group("alias")) - for match in _STAR_ALIAS.finditer(body) - if match.group("alias") in aliases - ] - if not starred and _BARE_STAR.search(body) is not None and len(tables) == 1: - starred.append((tables[0], "*")) - for table, alias in dict.fromkeys(starred): - found.append( - StarRead( - table=table, - alias=alias, - joined=tuple(other for other in tables if other != table), - location=location, - ) - ) - return found + for read in _star_reads_in_literal( + _LINE_COMMENT.sub(" ", literal.group("body")), f"{path}:{text.count(chr(10), 0, literal.start()) + 1}" + ) + ) def discover_star_reads(repo: Path, ref: str) -> tuple[StarRead, ...]: listing: Final = _git_or_none(repo, "grep", "-l", "--fixed-strings", TABLE_MARKER, ref, "--", *SOURCE_GLOBS) - if listing is None: - return () - found: Final[list[StarRead]] = [] - for line in listing.splitlines(): - path: Final = line.split(":", 1)[1] if ":" in line else line - text: Final = _git_or_none(repo, "show", f"{ref}:{path}") - if text is not None: - found.extend(star_reads_in_source(text, path)) - return tuple(found) + paths: Final = tuple(line.split(":", 1)[1] if ":" in line else line for line in (listing or "").splitlines()) + return tuple( + read + for path in paths + if (text := _git_or_none(repo, "show", f"{ref}:{path}")) is not None + for read in star_reads_in_source(text, path) + ) def _shape(read: StarRead) -> str: @@ -730,35 +720,46 @@ def _adds_not_null_without_default(statement: str) -> bool: ) +def _context_for( + statements: Sequence[str], + fresh_tables: frozenset[str], + star_reads: tuple[StarRead, ...], + schema: Schema | None, + defaulted: frozenset[tuple[str, str]], + base_unique: Mapping[str, tuple[str, tuple[str, ...]]], +) -> UpgradeContext: + dropped: Final = frozenset( + match.group("index") for statement in statements if (match := _DROP_INDEX_NAME.search(statement)) + ) + return UpgradeContext(fresh_tables, star_reads, schema, dropped, defaulted, base_unique) + + def build_report(repo: Path, base: str, head: str) -> Report: - base_names: Final = set(migration_names(repo, base)) - head_names: Final = migration_names(repo, head) - new_names: Final = tuple(name for name in head_names if name not in base_names) + base_names: Final = frozenset(migration_names(repo, base)) + new_names: Final = tuple(name for name in migration_names(repo, head) if name not in base_names) # The plans that break are the ones the old pods hold, so the queries that matter are the base version's. star_reads: Final = discover_star_reads(repo, base) - sql_by_migration: Final = {name: sql for name in new_names if (sql := read_migration(repo, head, name)) is not None} + statements_by_migration: Final = MappingProxyType( + {name: split_statements(sql) for name in new_names if (sql := read_migration(repo, head, name)) is not None} + ) created_here: Final = frozenset( match.group("table") - for sql in sql_by_migration.values() - for statement in split_statements(sql) + for statements in statements_by_migration.values() + for statement in statements if (match := _CREATE_TABLE.search(statement)) is not None ) # A re-declared `CREATE TABLE IF NOT EXISTS` of a table the old pods already use is not a new table. fresh_tables: Final = created_here - tables_created_before(repo, base) schema: Final = load_base_schema(repo, base) - statements_by_migration: Final = {name: split_statements(sql) for name, sql in sql_by_migration.items()} - defaulted: Final = _defaulted_columns([s for statements in statements_by_migration.values() for s in statements]) + defaulted: Final = _defaulted_columns(s for statements in statements_by_migration.values() for s in statements) base_unique: Final = unique_indexes_at(repo, base) - findings: Final[list[Finding]] = [] - for name, statements in statements_by_migration.items(): - dropped: Final = frozenset( - match.group("index") for statement in statements if (match := _DROP_INDEX_NAME.search(statement)) - ) - context: Final = UpgradeContext(fresh_tables, star_reads, schema, dropped, defaulted, base_unique) - findings.extend(classify(name, statement, context) for statement in statements) - + findings: Final = tuple( + classify(name, statement, _context_for(statements, fresh_tables, star_reads, schema, defaulted, base_unique)) + for name, statements in statements_by_migration.items() + for statement in statements + ) ranked: Final = sorted(findings, key=lambda finding: (SEVERITY_ORDER.index(finding.severity), finding.migration)) return Report( base=base, @@ -770,24 +771,26 @@ def build_report(repo: Path, base: str, head: str) -> Report: ) -PROCEDURE: Final = { - BREAKING: ( - "**Do not run the old and new versions side by side.** Take the previous version out of service before the " - "new version applies these migrations, or split the change across two releases." - ), - WRITE_REJECT: ( - "Writes from the previous version that break the new rule are rejected until those pods are gone. Confirm " - "the previous version already satisfies it on your data; if you cannot, do not overlap the two versions." - ), - PREPARED_PLAN: ( - "Expect the queries cited above to fail on pods still running the previous version until their pooled " - "connections are recreated. LiteLLM recreates them on the first such failure as of `1.98.0`; on older " - "versions, or to avoid the failures entirely, set `general_settings.database_disable_prepared_statements: " - "true` before upgrading, or drain the old pods before the new ones migrate." - ), - LOCK: "Apply these migrations in a maintenance window, or during a traffic trough, and watch for lock waits.", - INFO: "Safe to apply during a rolling upgrade.", -} +PROCEDURE: Final = MappingProxyType( + { + BREAKING: ( + "**Do not run the old and new versions side by side.** Take the previous version out of service before " + "the new version applies these migrations, or split the change across two releases." + ), + WRITE_REJECT: ( + "Writes from the previous version that break the new rule are rejected until those pods are gone. Confirm " + "the previous version already satisfies it on your data; if you cannot, do not overlap the two versions." + ), + PREPARED_PLAN: ( + "Expect the queries cited above to fail on pods still running the previous version until their pooled " + "connections are recreated. LiteLLM recreates them on the first such failure as of `1.98.0`; on older " + "versions, or to avoid the failures entirely, set `general_settings.database_disable_prepared_statements: " + "true` before upgrading, or drain the old pods before the new ones migrate." + ), + LOCK: "Apply these migrations in a maintenance window, or during a traffic trough, and watch for lock waits.", + INFO: "Safe to apply during a rolling upgrade.", + } +) def _cited_reads(report: Report) -> tuple[StarRead, ...]: @@ -797,55 +800,61 @@ def _cited_reads(report: Report) -> tuple[StarRead, ...]: return tuple(read for read in representatives if read is not None) +def _evidence_line(report: Report) -> str: + schema_note: Final = ( + f"no `schema.prisma` at `{report.base}`, so drops, renames and constraints are rated as if the previous " + "version used every column" + if report.schema_tables is None + else f"columns the previous version uses: `schema.prisma` at `{report.base}` ({report.schema_tables} tables)" + ) + cited: Final = _cited_reads(report) + shapes: Final = ", ".join(sorted({f"`{_shape(read)}` on `{read.table}` (`{read.location}`)" for read in cited})) + reads_note: Final = (f"whole-row reads: {shapes}",) if cited else () + return f"Evidence: {'; '.join((schema_note, *reads_note))}." + + def render_markdown(report: Report) -> str: - lines: Final[list[str]] = ["## Database Schema Changes", ""] if not report.migrations: - lines.append(f"No database schema changes since `{report.base}`.") - return "\n".join(lines) + "\n" + return f"## Database Schema Changes\n\nNo database schema changes since `{report.base}`.\n" count: Final = len(report.migrations) - plural: Final = "" if count == 1 else "s" - lines.extend( - [ - f"This release adds **{count} migration{plural}** to `litellm-proxy-extras`, applied by the first pod " - f"that boots (`prisma migrate deploy`). Rated against a pod still running `{report.base}`:", - "", - "| | Migration | Statement | Effect on pods still on the previous version |", - "|:--:|:--|:--|:--|", - ] + notable: Final = tuple(finding for finding in report.findings if finding.severity != INFO) + rows: Final = tuple( + f"| **{SEVERITY_LABEL[finding.severity]}** | `{finding.migration}` | `{_clip(finding.statement)}` | " + f"{finding.effect} |" + for finding in notable[:MAX_ROWS] ) - notable: Final = [finding for finding in report.findings if finding.severity != INFO] - for finding in notable[:MAX_ROWS]: - lines.append( - f"| **{SEVERITY_LABEL[finding.severity]}** | `{finding.migration}` | `{_clip(finding.statement)}` | " - f"{finding.effect} |" + overflow: Final = ( + ( + f"| | | | ...and {len(notable) - MAX_ROWS} more, worst first. Run " + f"`python3 ci_cd/migration_impact.py --base {report.base} --head {report.head}` for the full list. |", ) - if len(notable) > MAX_ROWS: - lines.append( - f"| | | | …and {len(notable) - MAX_ROWS} more, worst first. Run " - f"`python3 ci_cd/migration_impact.py --base {report.base} --head {report.head}` for the full list. |" - ) - + if len(notable) > MAX_ROWS + else () + ) worst: Final = report.worst - if worst is None or worst == INFO: - lines.append("| **INFO** | - | - | Nothing in this release affects a pod running the previous version. |") - lines.extend(["", PROCEDURE[worst or INFO], ""]) - evidence: Final[list[str]] = [] - if report.schema_tables is None: - evidence.append( - f"no `schema.prisma` at `{report.base}`, so drops, renames and constraints are rated as if the previous " - "version used every column" - ) - else: - evidence.append( - f"columns the previous version uses: `schema.prisma` at `{report.base}` ({report.schema_tables} tables)" - ) - cited: Final = _cited_reads(report) - if cited: - shapes: Final = ", ".join(sorted({f"`{_shape(read)}` on `{read.table}` (`{read.location}`)" for read in cited})) - evidence.append(f"whole-row reads: {shapes}") - lines.append(f"Evidence: {'; '.join(evidence)}.") - lines.append("") + quiet: Final = ( + ("| **INFO** | - | - | Nothing in this release affects a pod running the previous version. |",) + if worst is None or worst == INFO + else () + ) + lines: Final = ( + "## Database Schema Changes", + "", + f"This release adds **{count} migration{'' if count == 1 else 's'}** to `litellm-proxy-extras`, applied by " + f"the first pod that boots (`prisma migrate deploy`). Rated against a pod still running `{report.base}`:", + "", + "| | Migration | Statement | Effect on pods still on the previous version |", + "|:--:|:--|:--|:--|", + *rows, + *overflow, + *quiet, + "", + PROCEDURE[worst or INFO], + "", + _evidence_line(report), + "", + ) return "\n".join(lines) diff --git a/tests/test_litellm/test_migration_impact.py b/tests/test_litellm/test_migration_impact.py index 95aff2b8258..2f75db2528d 100644 --- a/tests/test_litellm/test_migration_impact.py +++ b/tests/test_litellm/test_migration_impact.py @@ -146,7 +146,7 @@ def test_split_statements_descends_into_dollar_quoted_blocks() -> None: def test_split_statements_drops_comments() -> None: statements = impact.split_statements('-- add a column\nALTER TABLE "T" ADD COLUMN "c" TEXT; /* trailing */') - assert statements == ['ALTER TABLE "T" ADD COLUMN "c" TEXT'] + assert statements == ('ALTER TABLE "T" ADD COLUMN "c" TEXT',) def test_star_read_discovery_records_table_alias_and_joins() -> None: @@ -165,7 +165,7 @@ def test_star_read_discovery_ignores_aliases_that_only_appear_in_a_comment() -> def test_star_read_discovery_ignores_view_definitions() -> None: # A view expands `*` when it is created, so it holds no per-connection prepared plan. - assert impact.star_reads_in_source(VIEW_SOURCE, "litellm/proxy/db/create_views.py") == [] + assert impact.star_reads_in_source(VIEW_SOURCE, "litellm/proxy/db/create_views.py") == () def test_add_column_on_a_whole_row_read_table_is_a_prepared_plan_change() -> None: @@ -269,7 +269,7 @@ def test_markdown_caps_the_table_and_says_where_the_rest_is() -> None: finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) rendered = impact.render_markdown(_report(findings=tuple([finding] * (impact.MAX_ROWS + 3)))) assert rendered.count("| **PREPARED-PLAN** |") == impact.MAX_ROWS - assert "…and 3 more" in rendered + assert "...and 3 more" in rendered def test_worst_severity_ranks_breaking_above_the_rest() -> None: From 95a9d4162689a09d4b2334046bbab7205dd3a4ef Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Mon, 26 Oct 2026 08:33:38 +0000 Subject: [PATCH 5/6] ci: run migration impact regressions in code quality checks Signed-off-by: Jaeyeon Kim --- .github/workflows/test-linting.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 3b9a2614725..5430b1245f9 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -536,6 +536,9 @@ jobs: - name: test_workflow_job_name_collisions run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py + - name: test_migration_impact + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_migration_impact.py + - name: test_unit_passed_gate run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py From 114128e5e304b77b048c4e69342ec518f251eaec Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Wed, 28 Oct 2026 09:21:53 +0000 Subject: [PATCH 6/6] fix: retain the worst effect of compound migration statements --- ci_cd/migration_impact.py | 16 +++++++--------- .../test_migration_impact.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/ci_cd/migration_impact.py b/ci_cd/migration_impact.py index d3e8167e1da..7cb372c69ae 100755 --- a/ci_cd/migration_impact.py +++ b/ci_cd/migration_impact.py @@ -572,6 +572,8 @@ def _rule_rename(site: _Site) -> Finding | None: def _rule_not_null(site: _Site) -> Finding | None: + if _ADD_COLUMN.search(site.statement) is not None and _adds_not_null_without_default(site.statement): + return site.rate(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") required: Final = [match.group("column") for match in _SET_NOT_NULL_COLUMN.finditer(site.statement)] if required: unguarded: Final = [ @@ -586,8 +588,6 @@ def _rule_not_null(site: _Site) -> Finding | None: "Those writes are now rejected.", ) return site.rate(INFO, "The previous version already always writes this column, so nothing changes for it.") - if _ADD_COLUMN.search(site.statement) is not None and _adds_not_null_without_default(site.statement): - return site.rate(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") return None @@ -687,9 +687,7 @@ def _rule_rest(site: _Site) -> Finding: return site.rate(INFO, "No effect on the previous version.") -# Worst-first: the first rule that has something to say about a statement decides its rating. RULES: Final = ( - _rule_new_table, _rule_drop, _rule_rename, _rule_not_null, @@ -704,11 +702,11 @@ def classify(migration: str, statement: str, context: UpgradeContext) -> Finding """Rate one statement by what it does to a pod still running the base version.""" table: Final = statement_table(statement) site: Final = _Site(migration, statement, table, context, _star_read_for(table, context.star_reads)) - for rule in RULES: - found: Final = rule(site) - if found is not None: - return found - return _rule_rest(site) + new_table: Final = _rule_new_table(site) + if new_table is not None: + return new_table + findings: Final = tuple(found for rule in RULES if (found := rule(site)) is not None) + return min(findings, key=lambda finding: SEVERITY_ORDER.index(finding.severity), default=_rule_rest(site)) def _adds_not_null_without_default(statement: str) -> bool: diff --git a/tests/code_coverage_tests/test_migration_impact.py b/tests/code_coverage_tests/test_migration_impact.py index 819677d609c..1e64c32cee6 100644 --- a/tests/code_coverage_tests/test_migration_impact.py +++ b/tests/code_coverage_tests/test_migration_impact.py @@ -13,6 +13,8 @@ from pathlib import Path from types import ModuleType from typing import Final +import pytest + ROOT: Final = Path(__file__).resolve().parents[2] CI_CD: Final = ROOT / "ci_cd" @@ -371,6 +373,23 @@ def test_not_null_is_judged_per_add_column_clause() -> None: assert one_default_is_not_enough.severity == impact.BREAKING +@pytest.mark.parametrize("benign", ['DROP COLUMN "unknown"', 'ALTER COLUMN "token" SET NOT NULL']) +@pytest.mark.parametrize( + "change,severity,effect", + [ + ('ADD COLUMN "required_new" TEXT NOT NULL', impact.BREAKING, "no default"), + ('ALTER COLUMN "token" TYPE INTEGER', impact.BREAKING, "to INTEGER"), + ('ADD COLUMN "optional_new" TEXT', impact.PREPARED_PLAN, "cached plan"), + ], +) +def test_compound_alter_keeps_the_worst_effect(benign: str, change: str, severity: str, effect: str) -> None: + statement: Final = f'ALTER TABLE "LiteLLM_VerificationToken" {benign}, {change}' + finding: Final = _classify(statement, schema=_base_schema(), reads=(_auth_read(),)) + assert finding.severity == severity + assert effect in finding.effect + assert finding.statement == statement + + def test_drop_not_null_beside_an_add_column_is_not_breaking() -> None: # 20250707230009_add_mcp_namespaced_tool_name: the `NOT NULL` here belongs to `DROP NOT NULL`. finding = _classify(