From e7332865356ec35d273bbf38a42b82df6b6387de Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:20:40 +0000 Subject: [PATCH] fix(migration_data_checks): keep the plain count when a fill reads an earlier write, DEFAULT, or fails to run --- .../migration_data_checks.py | 76 +++++++++++++------ .../migrations/test_request_log_indexes.py | 22 ++++-- .../test_migration_data_checks.py | 20 ++++- 3 files changed, 88 insertions(+), 30 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py index ec0b7974473..fbc93b16d7b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py @@ -127,10 +127,11 @@ _COMMENT: Final = re.compile(r"--[^\n]*") _NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE) _HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE) _UPDATE: Final = re.compile(rf"^\s*UPDATE\s+(?:ONLY\s+)?{_QUALIFIED}\s+SET\s+(.*)$", re.IGNORECASE | re.DOTALL) -_CLAUSE: Final = re.compile(r"\s(FROM|WHERE)\s", re.IGNORECASE) +_CLAUSE: Final = re.compile(r"\s(FROM|WHERE|RETURNING)\s", re.IGNORECASE) _ASSIGNMENT: Final = re.compile(rf"^{_IDENT}\s*=\s*(.*)$", re.DOTALL) _IS_NULL: Final = re.compile(rf"^{_IDENT}\s+IS\s+NULL$", re.IGNORECASE) _STRING_LITERAL: Final = re.compile(r"'[^']*'") +_IDENTIFIER: Final = re.compile(r'"([^"]+)"|\b([A-Za-z_]\w*)\b') def migration_data_check_timeout_seconds() -> float: @@ -269,23 +270,38 @@ def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]: class Update: """One UPDATE as the derivation reads it. `left_null` maps each assigned column to the predicate selecting the rows still NULL once the UPDATE ran, or to None when the shape - (a FROM, a WHERE other than that column IS NULL) can leave NULLs it cannot count. It is - None itself when the SET list cannot be read, so nothing is known about the table.""" + (a FROM, a WHERE other than that column IS NULL, a DEFAULT) can leave NULLs it cannot + count. It is None itself when the SET list cannot be read, so nothing is known about the table.""" table: str left_null: MappingProxyType[str, str | None] | None -Fills = MappingProxyType[tuple[str, str], str] +@dataclass(frozen=True, slots=True) +class Trail: + """What the UPDATEs read so far did: `fills` the predicate counting the rows still NULL per + (table, column), `written` every (table, column) one of them assigned, lower cased, with + (table, "*") standing for a SET list that could not be read.""" + + fills: MappingProxyType[tuple[str, str], str] + written: frozenset[tuple[str, str]] def _clause_start(text: str) -> int | None: - """Offset of the first FROM or WHERE outside parentheses and string literals, if any.""" + """Offset of the first FROM, WHERE or RETURNING outside parentheses and string literals.""" masked: Final = _mask_strings(text) depth: Final = _paren_depth(masked) return next((match.start() for match in _CLAUSE.finditer(masked) if depth[match.start()] == 0), None) +def _left_null_everywhere(value: str) -> str | None: + return None if value.upper() == "DEFAULT" else f"({value}) IS NULL" + + +def _left_null_where_null(column: str, value: str) -> str | None: + return None if value.upper() == "DEFAULT" else f'"{column}" IS NULL AND ({value}) IS NULL' + + def parse_update(statement_text: str) -> Update | None: update: Final = _UPDATE.match(statement_text) if update is None: @@ -296,16 +312,16 @@ def parse_update(statement_text: str) -> Update | None: if any(match is None for match in matches): return Update(table, None) assigned: Final = MappingProxyType({m.group(1): m.group(2).strip() for m in matches if m is not None}) - if cut is None: - return Update(table, MappingProxyType({column: f"({value}) IS NULL" for column, value in assigned.items()})) - clause: Final = rest[cut:].strip() + clause: Final = "" if cut is None else rest[cut:].strip() + if not clause or clause[:9].upper() == "RETURNING": + return Update(table, MappingProxyType({col: _left_null_everywhere(value) for col, value in assigned.items()})) is_null: Final = _IS_NULL.match(clause[5:].strip()) if clause[:5].upper() == "WHERE" else None filled: Final = is_null.group(1) if is_null is not None and is_null.group(1) in assigned else None return Update( table, MappingProxyType( { - column: f'"{column}" IS NULL AND ({value}) IS NULL' if column == filled else None + column: _left_null_where_null(column, value) if column == filled else None for column, value in assigned.items() } ), @@ -317,31 +333,47 @@ def _survives(key: tuple[str, str], update: Update) -> bool: return table != update.table or (update.left_null is not None and column not in update.left_null) -def _after(fills: Fills, update: Update | None) -> Fills: - """The fills still standing once `update` ran: what it leaves replaces the earlier entries - of the columns it writes, and an unreadable SET list drops every entry of its table.""" +def _reads_a_written_column(table: str, predicate: str, written: frozenset[tuple[str, str]]) -> bool: + names: Final = frozenset((quoted or bare).lower() for quoted, bare in _IDENTIFIER.findall(_mask_strings(predicate))) + return (table, "*") in written or any((table, name) in written for name in names) + + +def _after(trail: Trail, update: Update | None) -> Trail: + """The trail once `update` ran: what it leaves replaces the earlier fills of the columns it + writes, a fill that reads a column an earlier UPDATE wrote is not counted (the count would + read the old values), and an unreadable SET list drops every fill of its table.""" if update is None: - return fills - kept: Final = {key: fill for key, fill in fills.items() if _survives(key, update)} + return trail + columns: Final = MappingProxyType({}) if update.left_null is None else update.left_null + kept: Final = {key: fill for key, fill in trail.fills.items() if _survives(key, update)} added: Final = { - (update.table, column): fill for column, fill in (update.left_null or {}).items() if fill is not None + (update.table, column): fill + for column, fill in columns.items() + if fill is not None and not _reads_a_written_column(update.table, fill, trail.written) } - return MappingProxyType({**kept, **added}) + written: Final = ( + frozenset({(update.table, "*")}) + if update.left_null is None + else frozenset((update.table, column.lower()) for column in columns) + ) + return Trail(MappingProxyType({**kept, **added}), trail.written | written) -def _counting_what_is_left(check: Check, fills: Fills) -> Check: - if check.kind != "nulls" or (check.table, check.columns[0]) not in fills: +def _counting_what_is_left(check: Check, trail: Trail) -> Check: + if check.kind != "nulls" or (check.table, check.columns[0]) not in trail.fills: return check - return replace(check, expression=fills[(check.table, check.columns[0])]) + return replace(check, expression=trail.fills[(check.table, check.columns[0])]) def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]: """The checks of every statement in order. A `nulls` check counts the rows an earlier UPDATE of the same migration still leaves NULL, since `prisma migrate deploy` runs that UPDATE first.""" ordered: Final = statements(migration_sql) - fills: Final = tuple(accumulate((parse_update(text) for text in ordered), _after, initial=MappingProxyType({}))) + trails: Final = tuple( + accumulate((parse_update(text) for text in ordered), _after, initial=Trail(MappingProxyType({}), frozenset())) + ) return tuple( - _counting_what_is_left(check, fills[position]) + _counting_what_is_left(check, trails[position]) for position, statement_text in enumerate(ordered) for check in checks_for_statement(statement_text, f"migration {name}") ) # comprehension-ok: flattening the per-statement checks of one migration @@ -600,7 +632,7 @@ def _run_check( row: Final = connection.execute(_count_query(check, schema, types, limit_bytes)).fetchone() except psycopg.errors.QueryCanceled: return CouldNotVerify(check, timeout_seconds) - except psycopg.errors.UndefinedColumn: + except psycopg.errors.ProgrammingError: if check.kind != "nulls" or check.expression is None: raise return _run_check(connection, schema, replace(check, expression=None), types, limit_bytes, timeout_seconds) diff --git a/tests/integration/database/migrations/test_request_log_indexes.py b/tests/integration/database/migrations/test_request_log_indexes.py index 756b85d2900..1a2a072865d 100644 --- a/tests/integration/database/migrations/test_request_log_indexes.py +++ b/tests/integration/database/migrations/test_request_log_indexes.py @@ -1060,15 +1060,23 @@ def test_each_check_kind_counts_exactly_the_rows_postgres_would_reject(scratch_d @requires_db @pytest.mark.parametrize( - ("fill", "rows_still_null"), - (("COALESCE(\"d\", 'filled')", 0), ('"d"', 1)), - ids=("coalesce fills every row", "a nullable column leaves one row"), + ("prelude", "fill", "rows_counted", "set_not_null_error"), + ( + ("", "COALESCE(\"d\", 'filled')", 0, None), + ("", '"d"', 1, "NotNullViolation"), + ('ALTER TABLE "preflight_backfill" ADD COLUMN "n" TEXT DEFAULT \'new\';\n', '"n"', 2, None), + ), + ids=( + "coalesce fills every row", + "a nullable column leaves one row", + "a column the migration adds falls back to the plain count", + ), ) def test_a_nulls_check_after_a_backfill_counts_exactly_the_rows_postgres_rejects( - scratch_database: str, fill: str, rows_still_null: int + scratch_database: str, prelude: str, fill: str, rows_counted: int, set_not_null_error: str | None ) -> None: migration_sql: Final = ( - f'UPDATE "preflight_backfill" SET "c" = {fill} WHERE "c" IS NULL; -- data-migration-ok: small table\n' + prelude + f'UPDATE "preflight_backfill" SET "c" = {fill} WHERE "c" IS NULL; -- data-migration-ok: small table\n' 'ALTER TABLE "preflight_backfill" ALTER COLUMN "c" SET NOT NULL;\n' ) with psycopg.connect(scratch_database, autocommit=True) as conn: @@ -1083,9 +1091,9 @@ def test_a_nulls_check_after_a_backfill_counts_exactly_the_rows_postgres_rejects ) assert tuple(finding.rows for finding in findings if isinstance(finding, migration_data_checks.BlockedChange)) == ( - () if rows_still_null == 0 else (rows_still_null,) + () if rows_counted == 0 else (rows_counted,) ) - assert outcomes == (None, None if rows_still_null == 0 else "NotNullViolation") + assert outcomes == (*(None,) * (len(outcomes) - 1), set_not_null_error) def _apply(conn: "psycopg.Connection[tuple[object, ...]]", statement: str) -> str | None: diff --git a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py index d07bced2f37..70d3ed6ab90 100644 --- a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py +++ b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py @@ -110,6 +110,11 @@ def _left_null(migration_sql: str) -> tuple[str | None, ...]: ('UPDATE "T" SET "c" = COALESCE("d", \'x\'), "e" = 1; ' + SET_NOT_NULL, "(COALESCE(\"d\", 'x')) IS NULL"), ('UPDATE "T" SET "c" = NULL; ' + SET_NOT_NULL, "(NULL) IS NULL"), ('UPDATE "T" SET "c" = \'x\'; UPDATE "T" SET "c" = NULL; ' + SET_NOT_NULL, "(NULL) IS NULL"), + ( + 'UPDATE "T" SET "d" = \'x\'; UPDATE "T" SET "c" = \'y\' WHERE "c" IS NULL; ' + SET_NOT_NULL, + "\"c\" IS NULL AND ('y') IS NULL", + ), + ('UPDATE "T" SET "c" = \'x\' RETURNING "id"; ' + SET_NOT_NULL, "('x') IS NULL"), ( 'UPDATE "T" SET "c" = \'a, b WHERE 1\', "d" = 2 WHERE "c" IS NULL; ' + SET_NOT_NULL, "\"c\" IS NULL AND ('a, b WHERE 1') IS NULL", @@ -125,6 +130,8 @@ def _left_null(migration_sql: str) -> tuple[str | None, ...]: "coalesce no where", "set to null", "later write wins", + "earlier write to another column", + "returning", "quoted value", "subquery", ), @@ -146,6 +153,11 @@ def test_a_nulls_check_after_a_backfill_counts_the_rows_the_update_leaves_null( 'UPDATE "T" SET "c" = "O"."v" FROM "O" WHERE "O"."i" = "T"."i"; ' + SET_NOT_NULL, 'UPDATE "T" SET "c" = \'x\'; UPDATE "T" SET "c" = "d" WHERE "e" = 1; ' + SET_NOT_NULL, 'UPDATE "T" SET "c" = \'x\'; UPDATE "T" SET ("c", "d") = (1, 2); ' + SET_NOT_NULL, + 'UPDATE "T" SET "d" = \'x\'; UPDATE "T" SET "c" = "d" WHERE "c" IS NULL; ' + SET_NOT_NULL, + 'UPDATE "T" SET ("d", "e") = (1, 2); UPDATE "T" SET "c" = "d" WHERE "c" IS NULL; ' + SET_NOT_NULL, + 'UPDATE "T" SET "c" = DEFAULT; ' + SET_NOT_NULL, + 'UPDATE "T" SET "c" = DEFAULT WHERE "c" IS NULL; ' + SET_NOT_NULL, + 'UPDATE "T" SET "c" = \'x\' WHERE "c" IS NULL RETURNING "id"; ' + SET_NOT_NULL, ), ids=( "update after", @@ -156,6 +168,11 @@ def test_a_nulls_check_after_a_backfill_counts_the_rows_the_update_leaves_null( "update from", "later unreadable write", "later unreadable set list", + "fill reads a column written earlier", + "fill reads a table with an unreadable earlier write", + "default", + "default where is null", + "where is null then returning", ), ) def test_an_update_that_may_leave_nulls_it_cannot_count_keeps_the_plain_nulls_check(migration_sql: str) -> None: @@ -176,9 +193,10 @@ def test_an_update_that_may_leave_nulls_it_cannot_count_keeps_the_plain_nulls_ch {"c": "\"c\" IS NULL AND ('x') IS NULL", "d": None}, ), ('UPDATE ONLY "T" SET "c" = 1 WHERE "d" > 1', "T", {"c": None}), + ('UPDATE "T" SET "c" = DEFAULT, "d" = 1 RETURNING "id"', "T", {"c": None, "d": "(1) IS NULL"}), ('UPDATE "T" SET ("c", "d") = (1, 2)', "T", None), ), - ids=("no where", "where is null", "other where", "unreadable set list"), + ids=("no where", "where is null", "other where", "default and returning", "unreadable set list"), ) def test_an_update_names_what_it_leaves_null_per_column( statement: str, table: str, left_null: dict[str, str | None] | None