mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(migration_data_checks): skip the nulls check for a column an earlier UPDATE in the migration fills (LIT-10247)
This commit is contained in:
parent
aa769fbc69
commit
0ca7fe6d0d
3 changed files with 112 additions and 3 deletions
|
|
@ -18,7 +18,7 @@ import re
|
|||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import accumulate
|
||||
from itertools import accumulate, chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
|
@ -126,6 +126,11 @@ _PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST
|
|||
_COMMENT: Final = re.compile(r"--[^\n]*")
|
||||
_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
|
||||
_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
|
||||
_UPDATE: Final = re.compile(
|
||||
rf"^\s*UPDATE\s+(?:ONLY\s+)?{_QUALIFIED}\s+SET\s+(.*?)(?:\s+(FROM|WHERE)\s+(.*))?$", re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
_ASSIGNMENT: Final = re.compile(rf"^{_IDENT}\s*=\s*(.*)$", re.DOTALL)
|
||||
_IS_NULL: Final = re.compile(rf"^{_IDENT}\s+IS\s+NULL$", re.IGNORECASE)
|
||||
|
||||
|
||||
def migration_data_check_timeout_seconds() -> float:
|
||||
|
|
@ -248,11 +253,44 @@ def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
|
|||
return ()
|
||||
|
||||
|
||||
def backfilled_columns(statement_text: str) -> frozenset[tuple[str, str]]:
|
||||
"""The (table, column) pairs an UPDATE leaves without a NULL: every column it sets to
|
||||
something other than NULL when it has no WHERE, or that one column when its WHERE is
|
||||
`<column> IS NULL`. Any other shape, including an UPDATE ... FROM, fills nothing for sure."""
|
||||
update: Final = _UPDATE.match(statement_text)
|
||||
if update is None:
|
||||
return frozenset()
|
||||
table, assignments, clause, condition = update.groups()
|
||||
matches: Final = tuple(_ASSIGNMENT.match(assignment) for assignment in _split_top_level(assignments))
|
||||
filled: Final = frozenset(
|
||||
match.group(1) for match in matches if match is not None and match.group(2).strip().upper() != "NULL"
|
||||
)
|
||||
if clause is None:
|
||||
return frozenset((table, column) for column in filled)
|
||||
is_null: Final = _IS_NULL.match(condition.strip()) if clause.upper() == "WHERE" else None
|
||||
if is_null is None or is_null.group(1) not in filled:
|
||||
return frozenset()
|
||||
return frozenset({(table, is_null.group(1))})
|
||||
|
||||
|
||||
def _filled_before(ordered: tuple[str, ...], position: int) -> frozenset[tuple[str, str]]:
|
||||
return frozenset(chain.from_iterable(backfilled_columns(statement_text) for statement_text in ordered[:position]))
|
||||
|
||||
|
||||
def _checks_still_needed(statement_text: str, source: str, filled: frozenset[tuple[str, str]]) -> Iterator[Check]:
|
||||
for check in checks_for_statement(statement_text, source):
|
||||
if check.kind != "nulls" or (check.table, check.columns[0]) not in filled:
|
||||
yield check
|
||||
|
||||
|
||||
def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
|
||||
"""The checks of every statement, minus the `nulls` check of a column that an earlier
|
||||
UPDATE in the same migration fills, since `prisma migrate deploy` runs that UPDATE first."""
|
||||
ordered: Final = statements(migration_sql)
|
||||
return tuple(
|
||||
check
|
||||
for statement_text in statements(migration_sql)
|
||||
for check in checks_for_statement(statement_text, f"migration {name}")
|
||||
for position, statement_text in enumerate(ordered)
|
||||
for check in _checks_still_needed(statement_text, f"migration {name}", _filled_before(ordered, position))
|
||||
) # comprehension-ok: flattening the per-statement checks of one migration
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1058,6 +1058,29 @@ def test_each_check_kind_counts_exactly_the_rows_postgres_would_reject(scratch_d
|
|||
}
|
||||
|
||||
|
||||
_BACKFILL_THEN_SET_NOT_NULL: Final = (
|
||||
'UPDATE "preflight_backfill" SET "c" = \'filled\' WHERE "c" IS NULL; -- data-migration-ok: small table\n'
|
||||
'ALTER TABLE "preflight_backfill" ALTER COLUMN "c" SET NOT NULL;\n'
|
||||
)
|
||||
|
||||
|
||||
@requires_db
|
||||
def test_a_migration_that_fills_a_column_before_setting_it_not_null_passes_the_check_and_applies(
|
||||
scratch_database: str,
|
||||
) -> None:
|
||||
with psycopg.connect(scratch_database, autocommit=True) as conn:
|
||||
conn.execute('CREATE TABLE "preflight_backfill" ("id" INTEGER PRIMARY KEY, "c" TEXT)')
|
||||
conn.execute("INSERT INTO \"preflight_backfill\" VALUES (1, NULL), (2, 'kept')")
|
||||
checks: Final = migration_data_checks.checks_for_migration("probe", _BACKFILL_THEN_SET_NOT_NULL)
|
||||
findings: Final = migration_data_checks.find_blockers(conn, "public", checks)
|
||||
for statement in migration_data_checks.statements(_BACKFILL_THEN_SET_NOT_NULL):
|
||||
conn.execute(statement) # pyright: ignore[reportArgumentType] # test SQL
|
||||
values: Final = conn.execute('SELECT "c" FROM "preflight_backfill" ORDER BY "id"').fetchall()
|
||||
|
||||
assert findings == (), migration_data_checks.report(findings)
|
||||
assert values == [("filled",), ("kept",)]
|
||||
|
||||
|
||||
@requires_db
|
||||
@pytest.mark.parametrize(
|
||||
("index_columns", "s_value", "t_value", "longest_fitting_key", "described_as"),
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm_proxy_extras.migration_data_checks import (
|
|||
BlockedChange,
|
||||
Check,
|
||||
CouldNotVerify,
|
||||
backfilled_columns,
|
||||
btree_max_entry_bytes,
|
||||
checks_for_index,
|
||||
checks_for_migration,
|
||||
|
|
@ -88,6 +89,53 @@ def test_comments_and_blank_statements_are_dropped_before_matching() -> None:
|
|||
)
|
||||
|
||||
|
||||
BACKFILL_THEN_SET_NOT_NULL: Final = (
|
||||
'UPDATE "T" SET "c" = \'x\' WHERE "c" IS NULL; -- data-migration-ok: bounded by the rows of a small table\n'
|
||||
'ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL;\n'
|
||||
)
|
||||
|
||||
|
||||
def test_a_column_an_earlier_update_in_the_migration_fills_gets_no_nulls_check() -> None:
|
||||
assert _shape(checks_for_migration("x", BACKFILL_THEN_SET_NOT_NULL)) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"migration_sql",
|
||||
(
|
||||
'ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL; UPDATE "T" SET "c" = \'x\' WHERE "c" IS NULL;',
|
||||
'UPDATE "T" SET "c" = \'x\' WHERE "d" IS NULL; ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL;',
|
||||
'UPDATE "T" SET "c" = \'x\' WHERE "c" IS NULL AND "d" = 1; ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL;',
|
||||
'UPDATE "T" SET "c" = NULL; ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL;',
|
||||
'UPDATE "U" SET "c" = \'x\'; ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL;',
|
||||
'UPDATE "T" SET "c" = "O"."v" FROM "O" WHERE "O"."i" = "T"."i"; ALTER TABLE "T" ALTER COLUMN "c" SET NOT NULL;',
|
||||
),
|
||||
ids=("update after", "other column", "narrower where", "set to null", "other table", "update from"),
|
||||
)
|
||||
def test_an_update_that_may_leave_a_null_behind_keeps_the_nulls_check(migration_sql: str) -> None:
|
||||
assert _shape(checks_for_migration("x", migration_sql)) == (("nulls", "T", ("c",)),)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("statement", "expected"),
|
||||
(
|
||||
('UPDATE "T" SET "c" = \'x\' WHERE "c" IS NULL', frozenset({("T", "c")})),
|
||||
('UPDATE public."T" SET c = 1, d = now(), e = NULL', frozenset({("T", "c"), ("T", "d")})),
|
||||
('UPDATE "T" SET "c" = \'x\', "d" = 1 WHERE "c" IS NULL', frozenset({("T", "c")})),
|
||||
('UPDATE "T" SET ("c", "d") = (1, 2)', frozenset()),
|
||||
("SELECT 1", frozenset()),
|
||||
),
|
||||
)
|
||||
def test_an_update_names_the_columns_it_leaves_without_nulls(
|
||||
statement: str, expected: frozenset[tuple[str, str]]
|
||||
) -> None:
|
||||
assert backfilled_columns(statement) == expected
|
||||
|
||||
|
||||
def test_the_shipped_shadow_eval_backfill_keeps_only_its_index_check() -> None:
|
||||
migration_sql: Final = (MIGRATIONS / "20260817000000_shadow_eval_multi_key" / "migration.sql").read_text()
|
||||
assert _shape(checks_for_migration("x", migration_sql)) == (("too_long", "LiteLLM_ShadowEvalJob", ("group_id",)),)
|
||||
|
||||
|
||||
def test_every_shipped_migration_is_readable_and_the_later_indexes_on_text_columns_get_a_check() -> None:
|
||||
derived: Final = {
|
||||
folder.name: checks_for_migration(folder.name, (folder / "migration.sql").read_text())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue