From 886b59b56db36f1f5ce4b7e1b4eba858d888ce3d Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Fri, 18 Sep 2026 12:46:32 +0900 Subject: [PATCH] fix(ci): rate drops, renames, types and constraints on what the old client uses Review on #41744 found the ratings asserted more than they knew: every drop, rename and NOT NULL was breaking whether or not the old pods touched the column, a narrowing type change was rated as recoverable, and a new constraint could land as info because NOT VALID was read as "not enforced" when it only skips the scan of existing rows. The evidence now comes from the base version itself. schema.prisma at --base lists every column the old Prisma client selects and inserts, so dropping or renaming a column that schema never declared is info, a SET NOT NULL on a column the old client already always writes is info, and a type change is prepared-plan only when the new Postgres type is one the old Prisma type can still read (Int to BIGINT); anything else is breaking. Constraints get their own severity, write-reject, between breaking and prepared-plan: the old pods never enforced the rule, so rows they write may now be rejected, which is conditional on data rather than certain. A unique index that replaces one dropped in the same migration is only a lock. Without a schema at --base the old behaviour stands and the footnote says so. classify is split into ordered rule functions to stay under the complexity gate. Severity labels are text instead of emoji and the em dashes are gone, per the repository style rules. Verified on the full migration history: 7 breaking, all real. v1.99.0 to main adds one write-reject, the cascade foreign key on LiteLLM_JWTKeyMapping.token. Co-Authored-By: Claude Fable 5.1 --- ci_cd/migration_impact.py | 444 ++++++++++++++++---- litellm-proxy-extras/migration_runbook.md | 16 +- tests/test_litellm/test_migration_impact.py | 216 +++++++++- 3 files changed, 583 insertions(+), 93 deletions(-) diff --git a/ci_cd/migration_impact.py b/ci_cd/migration_impact.py index 6268d6939f6..9211e3314f5 100755 --- a/ci_cd/migration_impact.py +++ b/ci_cd/migration_impact.py @@ -12,20 +12,21 @@ This script diffs the migration set between two refs, classifies every statement to a pod still running the base version, and renders the result as markdown (for the release body) or JSON. Severities, worst first: -- `breaking` β€” the old pods cannot recover on their own: a column they read is gone, renamed - or retyped, or a constraint now rejects the rows they write -- `prepared-plan` β€” the change alters the result type of a query the old pods have prepared - (a `SELECT .*` gaining a column, or a column changing type), so the - plans cached on their pooled connections are rejected with - `cached plan must not change result type` until those connections are - recreated -- `lock` β€” the migration takes a blocking lock (or rewrites rows) on a table that is - already serving traffic -- `info` β€” no effect on the old pods (new tables, and changes to them) +- `breaking`: the old pods cannot recover on their own. A column they select is dropped, renamed + or changed to a type their client cannot read, or a NOT NULL now rejects the rows they write +- `write-reject`: a new constraint the old pods never enforced. Rows they write that violate it + are rejected; whether any do depends on the data +- `prepared-plan`: the result type of a query the old pods have prepared changes (a whole-row + `SELECT .*` gains a column, or a column widens), so the plans cached on their pooled + connections are rejected with `cached plan must not change result type` until those + connections are recreated +- `lock`: the migration takes a blocking lock, or rewrites rows, on a table already serving traffic +- `info`: no effect on the old pods (new tables, changes to them, columns they do not know) -The tables that are read through `SELECT .*` are discovered from the source tree at `--base` -(the code the old pods are running) rather than hardcoded, so the report keeps working as those -queries move. +The evidence for what the old pods read and write is the base version itself: `schema.prisma` at +`--base` lists every column the old Prisma client selects and inserts, and the source tree at +`--base` holds the raw `SELECT .*` queries. Nothing is hardcoded, so the report keeps +working as the schema and those queries move. python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 @@ -52,11 +53,34 @@ TABLE_MARKER: Final = 'FROM "' VIEW_MARKER: Final = re.compile(r"\bCREATE\s+(?:OR\s+REPLACE\s+)?(?:MATERIALIZED\s+)?VIEW\b", re.IGNORECASE) BREAKING: Final = "breaking" +WRITE_REJECT: Final = "write-reject" PREPARED_PLAN: Final = "prepared-plan" LOCK: Final = "lock" INFO: Final = "info" -SEVERITY_ORDER: Final = (BREAKING, PREPARED_PLAN, LOCK, INFO) -SEVERITY_ICON: Final = {BREAKING: "🚨", PREPARED_PLAN: "⚠️", LOCK: "πŸ”’", INFO: "ℹ️"} +SEVERITY_ORDER: Final = (BREAKING, WRITE_REJECT, PREPARED_PLAN, LOCK, INFO) +SEVERITY_LABEL: Final = { + BREAKING: "BREAKING", + WRITE_REJECT: "WRITE-REJECT", + PREPARED_PLAN: "PREPARED-PLAN", + LOCK: "LOCK", + INFO: "INFO", +} +SCHEMA_PATHS: Final = ("schema.prisma", "litellm/proxy/schema.prisma") +PRISMA_SCALARS: Final = frozenset( + {"String", "Int", "BigInt", "Float", "Decimal", "Boolean", "DateTime", "Json", "Bytes"} +) +# Postgres types a column of each Prisma type can become and still be read by the old client. +# Anything else is rated breaking: the plan is rejected once, and the retry cannot map the value. +WIDENING: Final = { + "Int": frozenset({"INT", "INT4", "INTEGER", "BIGINT", "INT8", "NUMERIC", "DECIMAL", "DOUBLE", "REAL", "FLOAT"}), + "BigInt": frozenset({"BIGINT", "INT8", "NUMERIC", "DECIMAL"}), + "Float": frozenset({"DOUBLE", "REAL", "FLOAT", "NUMERIC", "DECIMAL"}), + "Decimal": frozenset({"NUMERIC", "DECIMAL"}), + "String": frozenset({"TEXT", "VARCHAR", "CHARACTER", "CHAR", "CITEXT"}), + "DateTime": frozenset({"TIMESTAMP", "TIMESTAMPTZ"}), + "Json": frozenset({"JSON", "JSONB"}), + "Boolean": frozenset({"BOOLEAN", "BOOL"}), +} # Words that can follow a table name where an alias would sit, so they are never read as one. ALIAS_STOPWORDS: Final = frozenset( @@ -121,6 +145,36 @@ _DEFAULT: Final = re.compile(r"\bDEFAULT\b", re.IGNORECASE) _CONCURRENTLY: Final = re.compile(r"\bCONCURRENTLY\b", re.IGNORECASE) _ADD_CONSTRAINT: Final = re.compile(r"\bADD\s+CONSTRAINT\b", re.IGNORECASE) _NOT_VALID: Final = re.compile(r"\bNOT\s+VALID\b", re.IGNORECASE) +_UNIQUE_INDEX: Final = re.compile(r"\bCREATE\s+UNIQUE\s+INDEX\b", re.IGNORECASE) + +_DROP_COLUMN_NAME: Final = re.compile(r'\bDROP\s+COLUMN\s+(?:IF\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE) +_RENAME_COLUMN: Final = re.compile(r'\bRENAME\s+COLUMN\s+"?(?P\w+)"?\s+TO\b', re.IGNORECASE) +_RENAME_TABLE: Final = re.compile(r"\bRENAME\s+TO\b", re.IGNORECASE) +_SET_NOT_NULL_COLUMN: Final = re.compile( + r'\bALTER\s+(?:COLUMN\s+)?"?(?P\w+)"?\s+SET\s+NOT\s+NULL', re.IGNORECASE +) +_TYPE_CHANGE: Final = re.compile( + r'\bALTER\s+(?:COLUMN\s+)?"?(?P\w+)"?\s+(?:SET\s+DATA\s+)?TYPE\s+(?P\w+)', re.IGNORECASE +) +_CONSTRAINT_COLUMNS: Final = re.compile( + r"\b(?:FOREIGN\s+KEY|UNIQUE|PRIMARY\s+KEY)\s*\((?P[^)]*)\)", re.IGNORECASE +) +_CHECK_BODY: Final = re.compile(r"\bCHECK\s*\((?P.*)\)", re.IGNORECASE | re.DOTALL) +_INDEX_COLUMNS: Final = re.compile( + r'\bON\s+(?:ONLY\s+)?"?\w+"?\s*(?:USING\s+\w+\s*)?\((?P[^)]*)\)', re.IGNORECASE +) +_DROP_INDEX_NAME: Final = re.compile( + r'\bDROP\s+INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+EXISTS\s+)?"?(?P\w+)"?', re.IGNORECASE +) +_QUOTED_IDENTIFIER: Final = re.compile(r'"(?P\w+)"') + +_MODEL_BLOCK: Final = re.compile(r"^\s*model\s+(?P\w+)\s*\{(?P.*?)^\s*\}", re.MULTILINE | re.DOTALL) +_ENUM_NAME: Final = re.compile(r"^\s*enum\s+(?P\w+)", re.MULTILINE) +_PRISMA_FIELD: Final = re.compile( + r"^\s*(?P\w+)\s+(?P\w+)(?P\?)?(?P\[\])?(?P[^\n]*)$", re.MULTILINE +) +_FIELD_MAP: Final = re.compile(r'@map\("(?P[^"]+)"\)') +_TABLE_MAP: Final = re.compile(r'@@map\("(?P[^"]+)"\)') _SQL_LITERAL: Final = re.compile(r"(?P\"{3}|'{3})(?P.*?)(?P=quote)", re.DOTALL) _FROM_JOIN: Final = re.compile( @@ -142,6 +196,37 @@ class StarRead: location: str +@dataclass(frozen=True, slots=True) +class Column: + """One column as the base version's Prisma client knows it.""" + + required: bool + prisma_type: str + + +Schema = dict[str, dict[str, Column]] + + +@dataclass(frozen=True, slots=True) +class UpgradeContext: + """Everything a statement is rated against: what this upgrade creates, and what the old pods use.""" + + fresh_tables: frozenset[str] + star_reads: tuple[StarRead, ...] + schema: Schema | None + dropped_indexes: frozenset[str] = frozenset() + + def column(self, table: str, column: str) -> Column | None: + return None if self.schema is None else self.schema.get(table, {}).get(column) + + def knows_table(self, table: str) -> bool: + return self.schema is None or table in self.schema + + def knows_column(self, table: str, column: str) -> bool: + """Whether the old client selects or writes this column. Without a schema, assume it does.""" + return self.schema is None or self.column(table, column) is not None + + @dataclass(frozen=True, slots=True) class Finding: severity: str @@ -167,6 +252,7 @@ class Report: migrations: tuple[str, ...] findings: tuple[Finding, ...] star_reads: tuple[StarRead, ...] + schema_tables: int | None = None @property def worst(self) -> str | None: @@ -249,6 +335,59 @@ def tables_created_before(repo: Path, ref: str) -> frozenset[str]: return frozenset(match.group("table") for match in _CREATE_TABLE.finditer(listing or "")) +def parse_prisma_schema(text: str) -> Schema: + """The tables and columns a Prisma client generated from this schema selects and writes. + + Relation fields are not columns and are skipped; `@map` / `@@map` give the real names. + """ + enums: Final = frozenset(_ENUM_NAME.findall(text)) + schema: Final[Schema] = {} + for block in _MODEL_BLOCK.finditer(text): + body: Final = block.group("body") + table_map: Final = _TABLE_MAP.search(body) + table: Final = table_map.group("name") if table_map is not None else block.group("model") + columns: Final[dict[str, Column]] = {} + for field in _PRISMA_FIELD.finditer(body): + prisma_type: Final = field.group("type") + if prisma_type not in PRISMA_SCALARS and prisma_type not in enums: + continue + field_map: Final = _FIELD_MAP.search(field.group("rest")) + name: Final = field_map.group("name") if field_map is not None else field.group("name") + columns[name] = Column(required=field.group("optional") is None, prisma_type=prisma_type) + schema[table] = columns + return schema + + +def load_base_schema(repo: Path, ref: str) -> Schema | None: + for path in SCHEMA_PATHS: + text: Final = _git_or_none(repo, "show", f"{ref}:{path}") + if text is not None: + parsed: Final = parse_prisma_schema(text) + if parsed: + return parsed + return None + + +def _identifiers(fragment: str) -> tuple[str, ...]: + return tuple(dict.fromkeys(match.group("name") for match in _QUOTED_IDENTIFIER.finditer(fragment))) + + +def _constraint_columns(statement: str) -> tuple[str, ...]: + """The columns of this table that a new constraint checks on every write.""" + listed: Final = [ + name for match in _CONSTRAINT_COLUMNS.finditer(statement) for name in _identifiers(match.group("columns")) + ] + check: Final = _CHECK_BODY.search(statement) + if check is not None: + listed.extend(_identifiers(check.group("body"))) + return tuple(dict.fromkeys(listed)) + + +def _index_columns(statement: str) -> tuple[str, ...]: + match: Final = _INDEX_COLUMNS.search(statement) + return _identifiers(match.group("columns")) if match is not None else () + + def _alias_map(body: str) -> tuple[dict[str, str], tuple[str, ...]]: aliases: Final[dict[str, str]] = {} tables: Final[list[str]] = [] @@ -321,77 +460,204 @@ def _joined_read_for(table: str, star_reads: Sequence[StarRead]) -> StarRead | N return next((read for read in star_reads if table in read.joined), None) -def classify( - migration: str, - statement: str, - fresh_tables: frozenset[str], - star_reads: Sequence[StarRead], -) -> Finding: - """Rate one statement by what it does to a pod still running the base version.""" - table: Final = statement_table(statement) +@dataclass(frozen=True, slots=True) +class _Site: + """One statement under rating, with what the rules need to know about it.""" - def finding(severity: str, effect: str) -> Finding: - return Finding(severity=severity, migration=migration, table=table, statement=statement, effect=effect) + migration: str + statement: str + table: str + context: UpgradeContext + star: StarRead | None - if _CREATE_TABLE.search(statement) is not None: - return finding(INFO, "New table. The previous version does not read or write it.") - if table and table in fresh_tables: - return finding(INFO, "Table is created by this same upgrade, so no old pod reads or writes it yet.") + @property + def where(self) -> str: + return f" (`{self.star.location}`)" if self.star is not None else "" - star: Final = _star_read_for(table, star_reads) - where: Final = f" (`{star.location}`)" if star is not None else "" + def rate(self, severity: str, effect: str) -> Finding: + return Finding( + severity=severity, migration=self.migration, table=self.table, statement=self.statement, effect=effect + ) - if _DROP_COLUMN.search(statement) is not None or _DROP_TABLE.search(statement) is not None: - return finding( + def knows(self, column: str) -> bool: + return self.context.knows_column(self.table, column) + + +def _rule_new_table(site: _Site) -> Finding | None: + if _CREATE_TABLE.search(site.statement) is not None: + return site.rate(INFO, "New table. The previous version does not read or write it.") + if site.table and site.table in site.context.fresh_tables: + return site.rate(INFO, "Table is created by this same upgrade, so no old pod reads or writes it yet.") + return None + + +def _rule_drop(site: _Site) -> Finding | None: + if _DROP_TABLE.search(site.statement) is not None: + if site.context.knows_table(site.table): + return site.rate( + BREAKING, "Drops a table the previous version still queries. Old pods keep failing until they are gone." + ) + return site.rate(INFO, "Drops a table the previous version does not know.") + dropped: Final = _DROP_COLUMN_NAME.search(site.statement) + if dropped is None: + return None + column: Final = dropped.group("column") + if site.knows(column): + return site.rate( BREAKING, - f"Removes something the previous version still reads{where}. Old pods keep failing until they are gone; " - "reconnecting does not help.", + f"Drops `{column}`, which the previous version's client selects on every read of this table{site.where}. " + "Old pods keep failing until they are gone; reconnecting does not help.", ) - if _ALTER_TABLE.search(statement) is not None and _RENAME.search(statement) is not None: - return finding( + return site.rate(INFO, f"Drops `{column}`, a column the previous version does not know.") + + +def _rule_rename(site: _Site) -> Finding | None: + renamed: Final = _RENAME_COLUMN.search(site.statement) + if renamed is not None: + old_name: Final = renamed.group("column") + if site.knows(old_name): + return site.rate( + BREAKING, + f"Renames `{old_name}`, which the previous version still selects and writes by that name{site.where}. " + "Old pods cannot recover on their own.", + ) + return site.rate(INFO, f"Renames `{old_name}`, a column the previous version does not know.") + if _ALTER_TABLE.search(site.statement) is None or _RENAME_TABLE.search(site.statement) is None: + return None + if site.context.knows_table(site.table): + return site.rate(BREAKING, "Renames a table the previous version still queries by its old name.") + return site.rate(INFO, "Renames a table the previous version does not know.") + + +def _rule_not_null(site: _Site) -> Finding | None: + required: Final = [match.group("column") for match in _SET_NOT_NULL_COLUMN.finditer(site.statement)] + if required: + unguarded: Final = [ + column + for column in required + if (known := site.context.column(site.table, column)) is None or not known.required + ] + if unguarded or site.context.schema is None: + return site.rate( + BREAKING, + f"Requires `{'`, `'.join(unguarded or required)}`, which the previous version may write as NULL. " + "Those writes are now rejected.", + ) + return site.rate(INFO, "The previous version already always writes this column, so nothing changes for it.") + if _ADD_COLUMN.search(site.statement) is not None and _adds_not_null_without_default(site.statement): + return site.rate(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") + return None + + +def _rule_type_change(site: _Site) -> Finding | None: + changes: Final = list(_TYPE_CHANGE.finditer(site.statement)) + if not changes: + return None + narrowing: Final = [ + f"`{match.group('column')}` to {match.group('type').upper()}" + for match in changes + if (known := site.context.column(site.table, match.group("column"))) is None + or match.group("type").upper() not in WIDENING.get(known.prisma_type, frozenset()) + ] + if narrowing: + return site.rate( BREAKING, - f"Renames something the previous version still addresses by its old name{where}. Old pods cannot " - "recover on their own.", + f"Changes {', '.join(narrowing)}, which the previous version's client cannot be shown to read. Its " + "prepared plans are rejected once, and the retry may still fail to map the value.", ) - if _SET_NOT_NULL.search(statement) is not None: - return finding(BREAKING, "Rows the previous version writes without this column are now rejected.") - if _ADD_COLUMN.search(statement) is not None and _adds_not_null_without_default(statement): - return finding(BREAKING, "Adds a NOT NULL column with no default, so inserts from the previous version fail.") - if _ALTER_TYPE.search(statement) is not None: - return finding( - PREPARED_PLAN, - "Changes a column's type, so every prepared query on the previous version that returns it is rejected " - "with `cached plan must not change result type` until its connection is recreated. If the new type " - "is one the old client cannot map, it keeps failing after that.", + return site.rate( + PREPARED_PLAN, + "Widens a column's type. Every prepared query on the previous version that returns it is rejected with " + "`cached plan must not change result type` until its connection is recreated, then reads it fine.", + ) + + +def _rule_constraint(site: _Site) -> Finding | None: + if _ADD_CONSTRAINT.search(site.statement) is not None: + checked: Final = [column for column in _constraint_columns(site.statement) if site.knows(column)] + if checked: + return site.rate( + WRITE_REJECT, + f"Adds a rule on `{'`, `'.join(checked)}` that the previous version never enforced. Rows it writes " + "that violate the rule are now rejected; whether any do depends on the data.", + ) + if _NOT_VALID.search(site.statement) is not None: + return site.rate( + INFO, "Constraint on columns the previous version does not write, with no validation scan." + ) + return site.rate(LOCK, "Validates the constraint against every existing row while holding the table lock.") + if _UNIQUE_INDEX.search(site.statement) is None: + return None + covered: Final = _index_columns(site.statement) + replaces: Final = any(name.startswith(f"{site.table}_") for name in site.context.dropped_indexes) + if covered and all(site.knows(column) for column in covered) and not replaces: + return site.rate( + WRITE_REJECT, + f"Makes `{'`, `'.join(covered)}` unique, which the previous version never enforced. Duplicate rows it " + "writes are now rejected; whether any are depends on the data.", ) - if _ADD_COLUMN.search(statement) is not None and star is not None: - return finding( - PREPARED_PLAN, - f"Read whole-row by `{_shape(star)}`{where}, so the prepared plans cached on the previous version's " - "pooled connections are rejected with `cached plan must not change result type` until those " - "connections are recreated.", - ) - if _CREATE_INDEX.search(statement) is not None and _CONCURRENTLY.search(statement) is None: - joined: Final = _joined_read_for(table, star_reads) + return None + + +def _rule_whole_row_read(site: _Site) -> Finding | None: + if _ADD_COLUMN.search(site.statement) is None or site.star is None: + return None + return site.rate( + PREPARED_PLAN, + f"Read whole-row by `{_shape(site.star)}`{site.where}, so the prepared plans cached on the previous " + "version's pooled connections are rejected with `cached plan must not change result type` until those " + "connections are recreated.", + ) + + +def _rule_lock(site: _Site) -> Finding | None: + if _CREATE_INDEX.search(site.statement) is not None and _CONCURRENTLY.search(site.statement) is None: + joined: Final = _joined_read_for(site.table, site.context.star_reads) read_note: Final = ( f" The table is joined by `{_shape(joined)}` (`{joined.location}`), so that read waits too." if joined is not None else "" ) - return finding( + return site.rate( LOCK, "Blocks writes to the table (reads continue) until the index is built; the wait scales with the " f"table's size.{read_note}", ) - if _ADD_CONSTRAINT.search(statement) is not None and _NOT_VALID.search(statement) is None: - return finding(LOCK, "Validates the constraint against every existing row while holding the table lock.") - if _UPDATE.search(statement) is not None: - return finding(LOCK, "Rewrites existing rows, so it holds row locks for as long as the backfill runs.") - if _ADD_COLUMN.search(statement) is not None: - return finding(INFO, "Adds a nullable column to a table the previous version does not read whole-row.") - if _INSERT.search(statement) is not None: - return finding(INFO, "Inserts rows. The previous version is not affected.") - return finding(INFO, "No effect on the previous version.") + if _UPDATE.search(site.statement) is not None: + return site.rate(LOCK, "Rewrites existing rows, so it holds row locks for as long as the backfill runs.") + return None + + +def _rule_rest(site: _Site) -> Finding: + if _ADD_COLUMN.search(site.statement) is not None: + return site.rate(INFO, "Adds a nullable column to a table the previous version does not read whole-row.") + if _INSERT.search(site.statement) is not None: + return site.rate(INFO, "Inserts rows. The previous version is not affected.") + return site.rate(INFO, "No effect on the previous version.") + + +# Worst-first: the first rule that has something to say about a statement decides its rating. +RULES: Final = ( + _rule_new_table, + _rule_drop, + _rule_rename, + _rule_not_null, + _rule_type_change, + _rule_constraint, + _rule_whole_row_read, + _rule_lock, +) + + +def classify(migration: str, statement: str, context: UpgradeContext) -> Finding: + """Rate one statement by what it does to a pod still running the base version.""" + table: Final = statement_table(statement) + site: Final = _Site(migration, statement, table, context, _star_read_for(table, context.star_reads)) + for rule in RULES: + found: Final = rule(site) + if found is not None: + return found + return _rule_rest(site) def _adds_not_null_without_default(statement: str) -> bool: @@ -419,14 +685,26 @@ def build_report(repo: Path, base: str, head: str) -> Report: ) # A re-declared `CREATE TABLE IF NOT EXISTS` of a table the old pods already use is not a new table. fresh_tables: Final = created_here - tables_created_before(repo, base) + schema: Final = load_base_schema(repo, base) findings: Final[list[Finding]] = [] for name, sql in sql_by_migration.items(): - for statement in split_statements(sql): - findings.append(classify(name, statement, fresh_tables, star_reads)) + statements: Final = split_statements(sql) + dropped: Final = frozenset( + match.group("index") for statement in statements if (match := _DROP_INDEX_NAME.search(statement)) + ) + context: Final = UpgradeContext(fresh_tables, star_reads, schema, dropped) + findings.extend(classify(name, statement, context) for statement in statements) ranked: Final = sorted(findings, key=lambda finding: (SEVERITY_ORDER.index(finding.severity), finding.migration)) - return Report(base=base, head=head, migrations=new_names, findings=tuple(ranked), star_reads=star_reads) + return Report( + base=base, + head=head, + migrations=new_names, + findings=tuple(ranked), + star_reads=star_reads, + schema_tables=None if schema is None else len(schema), + ) PROCEDURE: Final = { @@ -434,6 +712,10 @@ PROCEDURE: Final = { "**Do not run the old and new versions side by side.** Take the previous version out of service before the " "new version applies these migrations, or split the change across two releases." ), + WRITE_REJECT: ( + "Writes from the previous version that break the new rule are rejected until those pods are gone. Confirm " + "the previous version already satisfies it on your data; if you cannot, do not overlap the two versions." + ), PREPARED_PLAN: ( "Expect the queries cited above to fail on pods still running the previous version until their pooled " "connections are recreated. LiteLLM recreates them on the first such failure as of `1.98.0`; on older " @@ -472,7 +754,7 @@ def render_markdown(report: Report) -> str: notable: Final = [finding for finding in report.findings if finding.severity != INFO] for finding in notable[:MAX_ROWS]: lines.append( - f"| {SEVERITY_ICON[finding.severity]} | `{finding.migration}` | `{_clip(finding.statement)}` | " + f"| **{SEVERITY_LABEL[finding.severity]}** | `{finding.migration}` | `{_clip(finding.statement)}` | " f"{finding.effect} |" ) if len(notable) > MAX_ROWS: @@ -483,12 +765,23 @@ def render_markdown(report: Report) -> str: worst: Final = report.worst if worst is None or worst == INFO: - lines.append("| ℹ️ | β€” | β€” | Nothing in this release affects a pod running the previous version. |") + lines.append("| **INFO** | - | - | Nothing in this release affects a pod running the previous version. |") lines.extend(["", PROCEDURE[worst or INFO], ""]) + evidence: Final[list[str]] = [] + if report.schema_tables is None: + evidence.append( + f"no `schema.prisma` at `{report.base}`, so drops, renames and constraints are rated as if the previous " + "version used every column" + ) + else: + evidence.append( + f"columns the previous version uses: `schema.prisma` at `{report.base}` ({report.schema_tables} tables)" + ) cited: Final = _cited_reads(report) if cited: shapes: Final = ", ".join(sorted({f"`{_shape(read)}` on `{read.table}` (`{read.location}`)" for read in cited})) - lines.append(f"Whole-row reads considered: {shapes}.") + evidence.append(f"whole-row reads: {shapes}") + lines.append(f"Evidence: {'; '.join(evidence)}.") lines.append("") return "\n".join(lines) @@ -513,6 +806,7 @@ def render_json(report: Report) -> str: "base": report.base, "head": report.head, "worst_severity": report.worst, + "schema_tables": report.schema_tables, "migrations": list(report.migrations), "findings": [finding.as_dict() for finding in report.findings], "star_reads": [ @@ -555,7 +849,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--output", help="write the report here instead of stdout") parser.add_argument( "--fail-on", - choices=(BREAKING, PREPARED_PLAN, LOCK), + choices=(BREAKING, WRITE_REJECT, PREPARED_PLAN, LOCK), help="exit 1 when a finding at this severity or worse is present", ) return parser.parse_args(argv) diff --git a/litellm-proxy-extras/migration_runbook.md b/litellm-proxy-extras/migration_runbook.md index 39cad08b303..788b9fc4428 100644 --- a/litellm-proxy-extras/migration_runbook.md +++ b/litellm-proxy-extras/migration_runbook.md @@ -130,13 +130,16 @@ migration set between two refs and rates every statement by what it does to thos | Severity | Meaning | |---|---| -| `breaking` | The old pods cannot recover on their own β€” a column they read is gone, renamed or retyped, or a constraint now rejects the rows they write | -| `prepared-plan` | The change alters the result type of a query the old pods have prepared - a `SELECT .*` gaining a column, or a column changing type - so the plans cached on their pooled connections are rejected with `cached plan must not change result type` until those connections are recreated ([#36418](https://github.com/BerriAI/litellm/issues/36418)) | +| `breaking` | The old pods cannot recover on their own. A column their client selects is dropped, renamed or changed to a type it cannot read, or a NOT NULL now rejects the rows they write | +| `write-reject` | A new constraint (foreign key, check, unique) on columns the old pods write. Rows they write that violate it are rejected; whether any do depends on the data | +| `prepared-plan` | The result type of a query the old pods have prepared changes: a whole-row `SELECT .*` gains a column, or a column widens. The plans cached on their pooled connections are rejected with `cached plan must not change result type` until those connections are recreated ([#36418](https://github.com/BerriAI/litellm/issues/36418)) | | `lock` | The migration takes a blocking lock, or rewrites rows, on a table already serving traffic | -| `info` | No effect on the old pods (new tables, and changes to them) | +| `info` | No effect on the old pods: new tables, changes to them, and columns the old pods do not know | -The tables read whole-row are discovered from the source tree at `--base` - the code the old pods -are running - not hardcoded, so the report follows those queries as they move. +The evidence is the base version itself, not a hardcoded list. `schema.prisma` at `--base` says +which columns the old Prisma client selects and writes, so dropping a column that schema never +declared is `info`, and dropping one it does declare is `breaking`. The source tree at `--base` +holds the raw `SELECT .*` queries whose plans go stale. The report's footnote names both. `create-release.yml` runs this and puts the result at the top of the release body. To see it for any pair of versions yourself: @@ -147,7 +150,8 @@ python3 ci_cd/migration_impact.py --base v1.93.0 --head v1.99.0 --format json ``` Standard library only, so it needs no `uv sync`. `--fail-on breaking` exits non-zero when a -migration in the range is not safe to apply under a rolling upgrade. +migration in the range is not safe to apply under a rolling upgrade; `--fail-on write-reject` +also stops on constraints the old pods might violate. --- diff --git a/tests/test_litellm/test_migration_impact.py b/tests/test_litellm/test_migration_impact.py index c23cd3627e7..7fa115e7318 100644 --- a/tests/test_litellm/test_migration_impact.py +++ b/tests/test_litellm/test_migration_impact.py @@ -72,8 +72,64 @@ def _auth_read() -> object: return reads[0] -def _classify(statement: str, fresh: frozenset[str] = frozenset(), reads: tuple[object, ...] = ()) -> object: - return impact.classify("20260713230852_add_key_type", statement, fresh, reads) +BASE_SCHEMA: Final = """ +enum KeyType { + llm_api + management +} + +model LiteLLM_VerificationToken { + token String @id + key_name String? + spend Float @default(0.0) + models String[] + team_id String? + key_type KeyType? + litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) + budget_id String? + @@index([team_id]) +} + +model LiteLLM_BudgetTable { + budget_id String @id + max_budget Float? + keys LiteLLM_VerificationToken[] +} + +model LiteLLM_ModelTable { + id Int @id @default(autoincrement()) + model_aliases Json? @map("aliases") + created_at DateTime @default(now()) @map("created_at") +} + +model ClaudeCodePlugin { + id String @id + @@map("LiteLLM_ClaudeCodePluginTable") +} +""" + + +def _context( + fresh: frozenset[str] = frozenset(), + reads: tuple[object, ...] = (), + schema: object = None, + dropped: frozenset[str] = frozenset(), +) -> object: + return impact.UpgradeContext(fresh, reads, schema, dropped) + + +def _classify( + statement: str, + fresh: frozenset[str] = frozenset(), + reads: tuple[object, ...] = (), + schema: object = None, + dropped: frozenset[str] = frozenset(), +) -> object: + return impact.classify("20260713230852_add_key_type", statement, _context(fresh, reads, schema, dropped)) + + +def _base_schema() -> object: + return impact.parse_prisma_schema(BASE_SCHEMA) def test_split_statements_descends_into_dollar_quoted_blocks() -> None: @@ -146,10 +202,29 @@ def test_index_builds_are_rated_by_whether_they_lock() -> None: assert concurrent.severity == impact.INFO -def test_unvalidated_constraint_is_not_a_lock() -> None: - validating = _classify('ALTER TABLE "LiteLLM_SpendLogs" ADD CONSTRAINT "fk" FOREIGN KEY ("t") REFERENCES "T"("id")') +def test_constraint_on_a_column_the_old_client_writes_can_reject_its_writes() -> None: + # NOT VALID only skips the scan of existing rows; new writes from the old pods are still checked. + schema = _base_schema() + checked = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "fk" FOREIGN KEY ("budget_id") ' + 'REFERENCES "LiteLLM_BudgetTable"("budget_id") NOT VALID', + schema=schema, + ) + assert checked.severity == impact.WRITE_REJECT + assert "`budget_id`" in checked.effect + + +def test_constraint_on_a_column_the_old_client_does_not_know_is_rated_by_its_scan() -> None: + schema = _base_schema() + validating = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "fk" FOREIGN KEY ("project_id") ' + 'REFERENCES "LiteLLM_ProjectTable"("project_id")', + schema=schema, + ) deferred = _classify( - 'ALTER TABLE "LiteLLM_SpendLogs" ADD CONSTRAINT "fk" FOREIGN KEY ("t") REFERENCES "T"("id") NOT VALID' + 'ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "fk" FOREIGN KEY ("project_id") ' + 'REFERENCES "LiteLLM_ProjectTable"("project_id") NOT VALID', + schema=schema, ) assert validating.severity == impact.LOCK assert deferred.severity == impact.INFO @@ -180,7 +255,7 @@ def test_markdown_leads_with_the_worst_finding_and_its_procedure() -> None: finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) rendered = impact.render_markdown(_report(findings=(finding,))) assert "20260713230852_add_key_type" in rendered - assert impact.SEVERITY_ICON[impact.PREPARED_PLAN] in rendered + assert "| **PREPARED-PLAN** |" in rendered assert impact.PROCEDURE[impact.PREPARED_PLAN] in rendered assert "`SELECT v.*` on `LiteLLM_VerificationToken`" in rendered @@ -188,7 +263,7 @@ def test_markdown_leads_with_the_worst_finding_and_its_procedure() -> None: def test_markdown_caps_the_table_and_says_where_the_rest_is() -> None: finding = _classify('ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT', reads=(_auth_read(),)) rendered = impact.render_markdown(_report(findings=tuple([finding] * (impact.MAX_ROWS + 3)))) - assert rendered.count("| ⚠️ |") == impact.MAX_ROWS + assert rendered.count("| **PREPARED-PLAN** |") == impact.MAX_ROWS assert "…and 3 more" in rendered @@ -247,7 +322,8 @@ def test_foreign_key_on_update_is_not_a_backfill() -> None: 'ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "fk" FOREIGN KEY ("token") ' 'REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE CASCADE ON UPDATE CASCADE NOT VALID' ) - assert finding.severity == impact.INFO + assert finding.severity == impact.WRITE_REJECT + assert "backfill" not in finding.effect backfill = _classify('UPDATE "LiteLLM_SpendLogs" SET "created_at" = "startTime" WHERE "created_at" IS NULL') assert backfill.severity == impact.LOCK @@ -257,12 +333,32 @@ def test_index_rename_is_not_breaking() -> None: assert finding.severity == impact.INFO -def test_type_change_is_a_prepared_plan_change_not_a_permanent_break() -> None: - finding = _classify('ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT') +SPEND_SCHEMA: Final = "model LiteLLM_DailyTagSpend {\n id String @id\n prompt_tokens Int\n}\n" + + +def test_widening_a_type_is_a_prepared_plan_change() -> None: + finding = _classify( + 'ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT', + schema=impact.parse_prisma_schema(SPEND_SCHEMA), + ) assert finding.severity == impact.PREPARED_PLAN assert "cached plan must not change result type" in finding.effect +def test_a_type_the_old_client_cannot_read_is_breaking() -> None: + finding = _classify( + 'ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE TEXT', + schema=impact.parse_prisma_schema(SPEND_SCHEMA), + ) + assert finding.severity == impact.BREAKING + assert "`prompt_tokens` to TEXT" in finding.effect + + +def test_type_change_without_a_schema_to_check_against_is_breaking() -> None: + finding = _classify('ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "prompt_tokens" SET DATA TYPE BIGINT') + assert finding.severity == impact.BREAKING + + def test_not_null_is_judged_per_add_column_clause() -> None: one_default_is_not_enough = _classify( 'ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "a" TEXT NOT NULL DEFAULT \'x\', ADD COLUMN "b" TEXT NOT NULL' @@ -325,6 +421,7 @@ def _two_version_repo(tmp_path: Path) -> Path: _git(repo, "config", "user.name", "test") (repo / "litellm" / "proxy").mkdir(parents=True) (repo / "litellm" / "proxy" / "utils.py").write_text(BASE_SOURCE, encoding="utf-8") + (repo / "schema.prisma").write_text(BASE_SCHEMA, encoding="utf-8") _write_migration( repo, "20250101000000_baseline", @@ -344,7 +441,11 @@ def _two_version_repo(tmp_path: Path) -> Path: 'CREATE TABLE IF NOT EXISTS "LiteLLM_VerificationToken" ("token" TEXT PRIMARY KEY);\n' 'ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "key_type" TEXT;\n' 'CREATE TABLE "LiteLLM_NewTable" ("id" TEXT PRIMARY KEY);\n' - 'CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id");\n', + 'CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id");\n' + # A column the base schema never declared: the old client cannot miss it. + 'ALTER TABLE "LiteLLM_TeamTable" DROP COLUMN IF EXISTS "legacy_flag";\n' + # A column it does declare: the old client selects it on every read. + 'ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name";\n', ) _git(repo, "add", ".") _git(repo, "commit", "-q", "-m", "head") @@ -366,7 +467,12 @@ def test_report_rates_migrations_against_the_queries_the_old_pods_run(tmp_path: assert add_column.severity == impact.PREPARED_PLAN new_index = by_statement['CREATE INDEX "new_idx" ON "LiteLLM_NewTable"("id")'] assert new_index.severity == impact.INFO - assert report.worst == impact.PREPARED_PLAN + unknown_drop = by_statement['ALTER TABLE "LiteLLM_TeamTable" DROP COLUMN IF EXISTS "legacy_flag"'] + assert unknown_drop.severity == impact.INFO + known_drop = by_statement['ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name"'] + assert known_drop.severity == impact.BREAKING + assert report.schema_tables == 4 + assert report.worst == impact.BREAKING def test_report_for_the_same_ref_twice_has_nothing_to_say(tmp_path: Path) -> None: @@ -375,3 +481,89 @@ def test_report_for_the_same_ref_twice_has_nothing_to_say(tmp_path: Path) -> Non assert report.migrations == () assert report.findings == () assert "No database schema changes since `v1.1.0`." in impact.render_markdown(report) + + +def test_schema_parser_reads_columns_maps_and_enums_and_skips_relations() -> None: + schema = _base_schema() + token = schema["LiteLLM_VerificationToken"] + assert token["token"] == impact.Column(required=True, prisma_type="String") + assert token["key_name"] == impact.Column(required=False, prisma_type="String") + assert token["models"].required # a list column is NOT NULL + assert token["key_type"].prisma_type == "KeyType" # enum-typed column + assert "litellm_budget_table" not in token # relation field, not a column + assert "aliases" in schema["LiteLLM_ModelTable"] # @map renames the column + assert "model_aliases" not in schema["LiteLLM_ModelTable"] + assert "LiteLLM_ClaudeCodePluginTable" in schema # @@map renames the table + assert "ClaudeCodePlugin" not in schema + + +def test_dropping_a_column_the_old_client_never_selects_is_not_breaking() -> None: + schema = _base_schema() + known = _classify('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "key_name"', schema=schema) + unknown = _classify('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN "settings_updated_at"', schema=schema) + assert known.severity == impact.BREAKING + assert "`key_name`" in known.effect + assert unknown.severity == impact.INFO + + +def test_renaming_a_column_is_rated_by_whether_the_old_client_uses_it() -> None: + schema = _base_schema() + known = _classify('ALTER TABLE "LiteLLM_VerificationToken" RENAME COLUMN "team_id" TO "owner_team"', schema=schema) + unknown = _classify('ALTER TABLE "LiteLLM_VerificationToken" RENAME COLUMN "tmp" TO "final"', schema=schema) + assert known.severity == impact.BREAKING + assert unknown.severity == impact.INFO + + +def test_set_not_null_is_breaking_only_where_the_old_client_may_write_null() -> None: + schema = _base_schema() + optional_at_base = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ALTER COLUMN "key_name" SET NOT NULL', schema=schema + ) + required_at_base = _classify( + 'ALTER TABLE "LiteLLM_VerificationToken" ALTER COLUMN "spend" SET NOT NULL', schema=schema + ) + assert optional_at_base.severity == impact.BREAKING + assert required_at_base.severity == impact.INFO + + +def test_dropping_a_table_the_old_client_does_not_know_is_informational() -> None: + schema = _base_schema() + known = _classify('DROP TABLE "LiteLLM_BudgetTable"', schema=schema) + unknown = _classify('DROP TABLE "LiteLLM_TmpTable"', schema=schema) + assert known.severity == impact.BREAKING + assert unknown.severity == impact.INFO + + +def test_unique_index_on_columns_the_old_client_writes_can_reject_duplicates() -> None: + finding = _classify( + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_key" ON "LiteLLM_VerificationToken"("key_name")', + schema=_base_schema(), + ) + assert finding.severity == impact.WRITE_REJECT + + +def test_unique_index_that_replaces_a_dropped_one_is_only_a_lock() -> None: + # The Daily*Spend migrations drop the old unique key and create a wider one in the same file. + finding = _classify( + 'CREATE UNIQUE INDEX "LiteLLM_VerificationToken_key_name_team_id_key" ON "LiteLLM_VerificationToken"' + '("key_name", "team_id")', + schema=_base_schema(), + dropped=frozenset({"LiteLLM_VerificationToken_key_name_key"}), + ) + assert finding.severity == impact.LOCK + + +def test_unique_index_on_a_new_column_is_only_a_lock() -> None: + finding = _classify( + 'CREATE UNIQUE INDEX "idx" ON "LiteLLM_VerificationToken"("settings_updated_at")', schema=_base_schema() + ) + assert finding.severity == impact.LOCK + + +def test_markdown_names_the_schema_it_used_as_evidence() -> None: + report = impact.Report( + base="v1.93.0", head="v1.99.0", migrations=("m",), findings=(), star_reads=(), schema_tables=65 + ) + assert "`schema.prisma` at `v1.93.0` (65 tables)" in impact.render_markdown(report) + blind = impact.Report(base="v1.93.0", head="v1.99.0", migrations=("m",), findings=(), star_reads=()) + assert "no `schema.prisma` at `v1.93.0`" in impact.render_markdown(blind)