From 6af13477e92531ebc1467becfda2430fe33f948d Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 00:05:36 +0000 Subject: [PATCH] test: read migrations from both sources Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- migrations/run.py | 2 +- tests/e2e/perf/setup/upgrade_worker.py | 6 +++--- .../migrations/test_db_schema_migration.py | 10 +++------- .../migrations/test_request_log_indexes.py | 11 +++++----- .../test_managed_file_flat_ids_index.py | 6 ++++-- .../database/test_migration_entrypoint.py | 6 ++++-- ...est_proxy_boot_without_general_settings.py | 4 ++-- .../test_request_log_indexes_at_boot.py | 6 ++++-- .../test_v1_migration_error_logging.py | 4 ++-- ...test_key_metadata_recovery_probe_bounds.py | 2 +- .../test_litellm_proxy_extras_utils.py | 1 + .../test_request_log_indexes.py | 4 ++-- .../test_check_migrations_no_data_rewrites.py | 20 ++++++++----------- 13 files changed, 41 insertions(+), 41 deletions(-) diff --git a/migrations/run.py b/migrations/run.py index 94e7cc7c0f0..c2b3924c087 100644 --- a/migrations/run.py +++ b/migrations/run.py @@ -4,7 +4,7 @@ Runs `prisma migrate deploy` against the LiteLLM writer database using the recovery logic in `litellm_proxy_extras.ProxyExtrasDBManager.setup_database` (P3005 baseline + P3009/P3018 idempotent-error handling, retries, etc.), then builds the request-log indexes the migrations leave out -(`litellm_proxy_extras.request_log_indexes`), waiting for them. The job exits +(`litellm_proxy_extras.spend_logs.request_log_indexes`), waiting for them. The job exits non-zero when an index could not be built so that it is rerun. A serving proxy that runs the migrations itself builds the same indexes in the background once it serves. diff --git a/tests/e2e/perf/setup/upgrade_worker.py b/tests/e2e/perf/setup/upgrade_worker.py index e69e841814f..7fa51b2258b 100644 --- a/tests/e2e/perf/setup/upgrade_worker.py +++ b/tests/e2e/perf/setup/upgrade_worker.py @@ -91,7 +91,8 @@ def wait_for_traffic(connection: psycopg.Connection[tuple[object, ...]], deadlin def main() -> int: assert os.environ.get("E2E_DATABASE_UPGRADE") == "1", "Only run against a disposable restored baseline" assert os.environ.get("DATABASE_PERF_CONFIRM_DISPOSABLE") == "1", "Disposable database confirmation required" - from litellm_proxy_extras import request_log_indexes + from litellm_proxy_extras.migration_sources import shipped_migrations + from litellm_proxy_extras.spend_logs import request_log_indexes from litellm_proxy_extras.utils import ProxyExtrasDBManager with connect() as connection: @@ -103,12 +104,11 @@ def main() -> int: ids: Final = tuple(row[0] for row in _STRINGS.validate_python(sample)) assert ids, "Restored baseline has no spend logs" names: Final = tuple(index.name for index in request_log_indexes.REQUEST_LOG_INDEXES) - migrations: Final = Path(request_log_indexes.__file__).parent / "migrations" initial: Final = UpgradeState( phase="schema", started_at=time.time(), required_indexes=names, - required_migrations=tuple(sorted(path.parent.name for path in migrations.glob("*/migration.sql"))), + required_migrations=tuple(migration.name for migration in shipped_migrations()), pending_indexes=missing_indexes(connection, names), baseline_rows=fingerprints(connection, ids), ) diff --git a/tests/integration/database/migrations/test_db_schema_migration.py b/tests/integration/database/migrations/test_db_schema_migration.py index 70498a3bcbd..13f9af82423 100644 --- a/tests/integration/database/migrations/test_db_schema_migration.py +++ b/tests/integration/database/migrations/test_db_schema_migration.py @@ -5,7 +5,8 @@ import tempfile from pathlib import Path import pytest -from litellm_proxy_extras.request_log_indexes import filter_request_log_index_diff +from litellm_proxy_extras.migration_sources import materialize_prisma_dir +from litellm_proxy_extras.spend_logs.request_log_indexes import filter_request_log_index_diff @pytest.mark.skipif( @@ -22,18 +23,13 @@ def test_schema_migration_in_sync(): so those statements are filtered out before the diff is judged. """ db_url = os.environ["DATABASE_URL"] - source_migrations_dir = Path( - "./litellm-proxy-extras/litellm_proxy_extras/migrations" - ) - source_schema_path = Path("./schema.prisma") temp_base = Path(tempfile.mkdtemp(prefix="litellm_schema_migration_")) schema_path = temp_base / "schema.prisma" migrations_dir = temp_base / "migrations" try: - shutil.copy(source_schema_path, schema_path) - shutil.copytree(source_migrations_dir, migrations_dir) + materialize_prisma_dir(temp_base) if not any(migrations_dir.iterdir()): pytest.fail( diff --git a/tests/integration/database/migrations/test_request_log_indexes.py b/tests/integration/database/migrations/test_request_log_indexes.py index 3b6d1eaf21a..e2fb57eca48 100644 --- a/tests/integration/database/migrations/test_request_log_indexes.py +++ b/tests/integration/database/migrations/test_request_log_indexes.py @@ -14,10 +14,11 @@ from typing import Final import psycopg import pytest -from litellm_proxy_extras import request_log_indexes +from litellm_proxy_extras.migration_sources import shipped_migrations +from litellm_proxy_extras.spend_logs import request_log_indexes from litellm_proxy_extras.migration_lock import MIGRATION_LOCK_KEY, migration_lock from litellm_proxy_extras.migration_recovery import roll_back_failed_inert_migration -from litellm_proxy_extras.request_log_indexes import ( +from litellm_proxy_extras.spend_logs.request_log_indexes import ( REQUEST_LOG_INDEXES, RequestLogIndex, build_index_on_partitioned_table, @@ -91,8 +92,8 @@ def _release_layout(prisma_dir: Path, before: str) -> Path: the way the releases that shipped them did: the Prisma layout of a proxy on that release.""" (prisma_dir / "migrations").mkdir(parents=True) shutil.copy(PACKAGE / "schema.prisma", prisma_dir / "schema.prisma") - for migration in sorted((PACKAGE / "migrations").iterdir()): - if migration.is_dir() and migration.name < before: + for migration in shipped_migrations(): + if migration.name < before: shutil.copytree(migration, prisma_dir / "migrations" / migration.name) for name, original in ORIGINAL_MIGRATION_SQL.items(): if (prisma_dir / "migrations" / name).is_dir(): @@ -399,7 +400,7 @@ def test_a_failed_row_whose_migration_still_runs_sql_in_this_build_is_left_for_t @requires_db def test_a_migration_without_a_failed_row_is_not_touched(partitioned_database: str) -> None: - inert: Final = PACKAGE / "migrations" / CALL_ID_INDEX_MIGRATION / "migration.sql" + inert: Final = PACKAGE / "spend_logs" / "migrations" / CALL_ID_INDEX_MIGRATION / "migration.sql" before: Final = _ledger(partitioned_database) with migration_lock(partitioned_database) as coordinator: diff --git a/tests/integration/database/test_managed_file_flat_ids_index.py b/tests/integration/database/test_managed_file_flat_ids_index.py index b297887411f..b3d13a8d33f 100644 --- a/tests/integration/database/test_managed_file_flat_ids_index.py +++ b/tests/integration/database/test_managed_file_flat_ids_index.py @@ -17,12 +17,14 @@ from integration._support.process import owned_proxy, proxy_database_environment from integration._support.wire import Reply, Request, wire_server from psycopg import sql from pydantic import JsonValue, TypeAdapter +from litellm_proxy_extras.migration_sources import shipped_migrations REPO_ROOT: Final = Path(__file__).resolve().parents[3] PRISMA_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" GIN_MIGRATION: Final = "20261003000000_add_managed_file_flat_ids_gin_index" INDEX_NAME: Final = "LiteLLM_ManagedFileTable_flat_model_file_ids_idx" -SHIPPED_MIGRATIONS: Final = tuple(sorted(path.name for path in (PRISMA_DIR / "migrations").iterdir() if path.is_dir())) +SHIPPED: Final = {migration.name: migration for migration in shipped_migrations()} +SHIPPED_MIGRATIONS: Final = tuple(SHIPPED) INDEX_ROW: Final = ( "SELECT i.indexdef, x.indisvalid FROM pg_indexes i " "JOIN pg_class c ON c.relname = i.indexname JOIN pg_index x ON x.indexrelid = c.oid WHERE i.indexname = %s" @@ -70,7 +72,7 @@ def _deploy_schema_before(database_url: str, directory: Path, migration: str) -> shutil.copy(PRISMA_DIR / "schema.prisma", older / "schema.prisma") shutil.copy(PRISMA_DIR / "migrations" / "migration_lock.toml", older / "migrations" / "migration_lock.toml") for name in (name for name in SHIPPED_MIGRATIONS if name < migration): - shutil.copytree(PRISMA_DIR / "migrations" / name, older / "migrations" / name) + shutil.copytree(SHIPPED[name], older / "migrations" / name) subprocess.run( [sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(older / "schema.prisma")], check=True, diff --git a/tests/integration/database/test_migration_entrypoint.py b/tests/integration/database/test_migration_entrypoint.py index 19c480b9531..3d5c3fa8816 100644 --- a/tests/integration/database/test_migration_entrypoint.py +++ b/tests/integration/database/test_migration_entrypoint.py @@ -15,11 +15,13 @@ from integration._support.client import Gateway from integration._support.process import owned_proxy from psycopg import sql from psycopg.rows import dict_row +from litellm_proxy_extras.migration_sources import shipped_migrations REPO_ROOT: Final = Path(__file__).resolve().parents[3] PRISMA_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" MISSING_MIGRATION: Final = "20260626120000_add_mcp_tool_search_enabled" -SHIPPED_MIGRATIONS: Final = tuple(sorted(path.name for path in (PRISMA_DIR / "migrations").iterdir() if path.is_dir())) +SHIPPED: Final = {migration.name: migration for migration in shipped_migrations()} +SHIPPED_MIGRATIONS: Final = tuple(SHIPPED) @contextmanager @@ -41,7 +43,7 @@ def deploy_older_schema(database_url: str, directory: Path) -> None: shutil.copy(PRISMA_DIR / "schema.prisma", older / "schema.prisma") shutil.copy(PRISMA_DIR / "migrations" / "migration_lock.toml", older / "migrations" / "migration_lock.toml") for name in (name for name in SHIPPED_MIGRATIONS if name < MISSING_MIGRATION): - shutil.copytree(PRISMA_DIR / "migrations" / name, older / "migrations" / name) + shutil.copytree(SHIPPED[name], older / "migrations" / name) subprocess.run( [sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(older / "schema.prisma")], check=True, diff --git a/tests/integration/database/test_proxy_boot_without_general_settings.py b/tests/integration/database/test_proxy_boot_without_general_settings.py index 73254f3862f..3f0d30ddbe6 100644 --- a/tests/integration/database/test_proxy_boot_without_general_settings.py +++ b/tests/integration/database/test_proxy_boot_without_general_settings.py @@ -8,12 +8,12 @@ from pydantic import JsonValue from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value, string_value from integration._support.database import read_rows, scratch_database from integration._support.process import DB_PUSH, LEGACY_MIGRATE_DEPLOY, MIGRATE_DEPLOY, owned_proxy_process +from litellm_proxy_extras.migration_sources import shipped_migrations pytestmark: Final = pytest.mark.timeout(300) REPO_ROOT: Final = Path(__file__).resolve().parents[3] -MIGRATIONS_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations" -SHIPPED_MIGRATIONS: Final = frozenset(path.name for path in MIGRATIONS_DIR.iterdir() if path.is_dir()) +SHIPPED_MIGRATIONS: Final = frozenset(migration.name for migration in shipped_migrations()) CANNED_REPLY: Final = "Hello! This is a mock response from the fake OpenAI endpoint." PROVIDER_KEY: Final = "integration-provider-key" SCRATCH_ONLY: Final = ("DATABASE_URL_READ_REPLICA",) diff --git a/tests/integration/database/test_request_log_indexes_at_boot.py b/tests/integration/database/test_request_log_indexes_at_boot.py index e72477de901..b6b6504bfa8 100644 --- a/tests/integration/database/test_request_log_indexes_at_boot.py +++ b/tests/integration/database/test_request_log_indexes_at_boot.py @@ -16,13 +16,15 @@ from integration._support.database import scratch_database from integration._support.process import LEGACY_MIGRATE_DEPLOY, MIGRATE_DEPLOY, owned_proxy_process from psycopg import sql from psycopg.rows import class_row +from litellm_proxy_extras.migration_sources import shipped_migrations pytestmark: Final = pytest.mark.timeout(240) REPO_ROOT: Final = Path(__file__).resolve().parents[3] PRISMA_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" PARTITION_SCRIPT: Final = REPO_ROOT / "db_scripts" / "partition_spend_logs.sql" -SHIPPED_MIGRATIONS: Final = tuple(sorted(path.name for path in (PRISMA_DIR / "migrations").iterdir() if path.is_dir())) +SHIPPED: Final = {migration.name: migration for migration in shipped_migrations()} +SHIPPED_MIGRATIONS: Final = tuple(SHIPPED) API_KEY_INDEX_MIGRATION: Final = "20260823000000_add_spend_logs_api_key_starttime_index" CALL_ID_INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index" ORIGINAL_MIGRATION_SQL: Final = MappingProxyType( @@ -76,7 +78,7 @@ def release_layout(directory: Path, migrations: tuple[str, ...]) -> Path: shutil.copy(PRISMA_DIR / "schema.prisma", directory / "schema.prisma") shutil.copy(PRISMA_DIR / "migrations" / "migration_lock.toml", directory / "migrations" / "migration_lock.toml") for name in migrations: - shutil.copytree(PRISMA_DIR / "migrations" / name, directory / "migrations" / name) + shutil.copytree(SHIPPED[name], directory / "migrations" / name) for name, original in ORIGINAL_MIGRATION_SQL.items(): if name in migrations: (directory / "migrations" / name / "migration.sql").write_text(original) diff --git a/tests/integration/database/test_v1_migration_error_logging.py b/tests/integration/database/test_v1_migration_error_logging.py index 3ca2179d983..7edcf2ff7e4 100644 --- a/tests/integration/database/test_v1_migration_error_logging.py +++ b/tests/integration/database/test_v1_migration_error_logging.py @@ -27,15 +27,15 @@ from psycopg.types.json import Jsonb from tests.integration._support.client import eventually from tests.integration._support.ports import allocate_port from tests.integration._support.process import stop_process_group +from litellm_proxy_extras.migration_sources import shipped_migrations REPO_ROOT: Final = Path(__file__).resolve().parents[3] -MIGRATIONS_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations" MIGRATION_NAME: Final = "20260921190000_agent_identity" pytestmark: Final = pytest.mark.timeout(300) PASSWORD: Final = "wr ong'pw9" FRAGMENTS: Final = ("wr ong", "wr+ong", "ong'pw9", "wr%20ong", "ong%27pw9", "pw9") SHIPPED_MIGRATIONS: Final = tuple( - sorted(path.name for path in MIGRATIONS_DIR.iterdir() if path.is_dir() and path.name != "0_init") + migration.name for migration in shipped_migrations() if migration.name != "0_init" ) LOG_PREFIX: Final = r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - [^\n]+ - " LOG_RECORD_START: Final = rf"{LOG_PREFIX}(?:DEBUG|INFO|WARNING|ERROR|CRITICAL) - " diff --git a/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py b/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py index fd67721cc6b..8d479a5260a 100644 --- a/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py +++ b/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py @@ -5,7 +5,7 @@ from typing import Final import psycopg import pytest -from litellm_proxy_extras.request_log_indexes import REQUEST_LOG_INDEXES +from litellm_proxy_extras.spend_logs.request_log_indexes import REQUEST_LOG_INDEXES from psycopg.types.json import Jsonb from pydantic import JsonValue diff --git a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py index 02bdf3429ea..3efb025b1ca 100644 --- a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py +++ b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py @@ -281,6 +281,7 @@ def _get_all_migrations(): """Return (migration_name, sql_content) pairs for all migrations.""" migration_files = sorted( glob.glob(os.path.join(_MIGRATIONS_DIR, "*/migration.sql")) + + glob.glob(os.path.join(_MIGRATIONS_DIR, "..", "spend_logs", "migrations", "*/migration.sql")) ) results = [] for path in migration_files: diff --git a/tests/unit/litellm_proxy_extras/test_request_log_indexes.py b/tests/unit/litellm_proxy_extras/test_request_log_indexes.py index 5cf7593c5cd..0629c533016 100644 --- a/tests/unit/litellm_proxy_extras/test_request_log_indexes.py +++ b/tests/unit/litellm_proxy_extras/test_request_log_indexes.py @@ -4,7 +4,7 @@ from typing import Final import pytest from litellm_proxy_extras.migration_recovery import is_inert_migration -from litellm_proxy_extras.request_log_indexes import ( +from litellm_proxy_extras.spend_logs.request_log_indexes import ( REQUEST_LOG_INDEXES, RequestLogIndex, filter_request_log_index_diff, @@ -41,7 +41,7 @@ class TestTheIndexList: @pytest.mark.parametrize("name", INERT_MIGRATIONS) def test_the_migrations_that_used_to_build_these_indexes_run_no_sql(self, name: str): - assert is_inert_migration((PACKAGE / "migrations" / name / "migration.sql").read_text()) + assert is_inert_migration((PACKAGE / "spend_logs" / "migrations" / name / "migration.sql").read_text()) class TestIsInertMigration: diff --git a/tests/unit/test_check_migrations_no_data_rewrites.py b/tests/unit/test_check_migrations_no_data_rewrites.py index fb1053ff887..b970b56bbc9 100644 --- a/tests/unit/test_check_migrations_no_data_rewrites.py +++ b/tests/unit/test_check_migrations_no_data_rewrites.py @@ -210,7 +210,7 @@ class TestDefaultedColumnsOnRequestLogTables: class TestIndexesOnLogTables: """Every CREATE INDEX on a request-log table is rejected: a plain one blocks writes for the whole build and a concurrent one fails on a partitioned parent, so the migration job - (litellm_proxy_extras/request_log_indexes.py) builds those instead.""" + (litellm_proxy_extras/spend_logs/request_log_indexes.py) builds those instead.""" def test_the_original_spend_log_index_statement_is_flagged(self, tmp_path): sql = ( @@ -286,7 +286,7 @@ class TestIndexesOnLogTables: rendered = _scan(tmp_path, sql)[0].render() assert "20260101000000_fixture/migration.sql:1" in rendered assert "blocks writes until the build finishes, or fails on a partitioned table" in rendered - assert "REQUEST_LOG_INDEXES in litellm_proxy_extras/request_log_indexes.py" in rendered + assert "REQUEST_LOG_INDEXES in litellm_proxy_extras/spend_logs/request_log_indexes.py" in rendered @pytest.mark.parametrize( "name", @@ -1829,8 +1829,8 @@ class TestReporting: class TestGrandfathering: def test_every_grandfathered_migration_still_violates(self): for name in sorted(checker.GRANDFATHERED): - directory = checker.MIGRATIONS_DIR / name - assert directory.is_dir(), f"{name} no longer exists; drop it from GRANDFATHERED" + directory = next((s / name for s in checker.MIGRATION_SOURCES if (s / name).is_dir()), None) + assert directory is not None, f"{name} no longer exists; drop it from GRANDFATHERED" assert checker.scan_migration(directory), f"{name} is clean; drop it from GRANDFATHERED" def test_stale_entry_is_reported_when_a_migration_stops_violating(self): @@ -1841,11 +1841,7 @@ class TestGrandfathering: assert checker.stale_grandfathers({}) == tuple(sorted(checker.GRANDFATHERED)) def test_no_stale_entries_against_the_real_tree(self): - found = { - path.name: checker.scan_migration(path) - for path in checker.MIGRATIONS_DIR.iterdir() - if (path / "migration.sql").is_file() - } + found = {path.name: checker.scan_migration(path) for path in checker.shipped_migrations()} assert checker.stale_grandfathers(found) == () @@ -1867,7 +1863,7 @@ def _tree(monkeypatch, tmp_path: Path, sql: str, grandfathered: frozenset = froz directory.mkdir(parents=True) (directory / "migration.sql").write_text(sql, encoding="utf-8") monkeypatch.setattr(checker, "REPO_ROOT", tmp_path) - monkeypatch.setattr(checker, "MIGRATIONS_DIR", tmp_path / "migrations") + monkeypatch.setattr(checker, "MIGRATION_SOURCES", (tmp_path / "migrations",)) monkeypatch.setattr(checker, "GRANDFATHERED", grandfathered) @@ -1889,7 +1885,7 @@ class TestExitCode: assert checker.main() == 0 def test_a_missing_migrations_directory_is_an_error(self, tmp_path, monkeypatch): - monkeypatch.setattr(checker, "MIGRATIONS_DIR", tmp_path / "absent") + monkeypatch.setattr(checker, "MIGRATION_SOURCES", (tmp_path / "absent",)) assert checker.main() == 2 def test_the_failure_names_the_migration_the_line_and_the_keyword( @@ -1908,6 +1904,6 @@ class TestExitCode: assert f"{FIXTURE}: listed in GRANDFATHERED" in capsys.readouterr().out def test_a_missing_directory_is_reported_on_stderr(self, tmp_path, monkeypatch, capsys): - monkeypatch.setattr(checker, "MIGRATIONS_DIR", tmp_path / "absent") + monkeypatch.setattr(checker, "MIGRATION_SOURCES", (tmp_path / "absent",)) checker.main() assert "migrations directory not found" in capsys.readouterr().err