diff --git a/.circleci/scripts/run_migration_tests.py b/.circleci/scripts/run_migration_tests.py index 07f3c66a280..5a73c54e3f5 100644 --- a/.circleci/scripts/run_migration_tests.py +++ b/.circleci/scripts/run_migration_tests.py @@ -14,7 +14,7 @@ SUITES: Final = { "recovery": (("test_recovery.py",), 15), "legacy": (("test_legacy.py", "test_pooling.py"), 11), "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5), - "shaped": (("test_shaped_database.py",), 2), + "shaped": (("test_shaped_database.py",), 1), } diff --git a/tests/e2e/migrations/database.py b/tests/e2e/migrations/database.py index caca348593a..a370c21ba0b 100644 --- a/tests/e2e/migrations/database.py +++ b/tests/e2e/migrations/database.py @@ -42,7 +42,7 @@ class Database: connection.execute("SET statement_timeout = '15s'") yield connection - def execute(self, statement: LiteralString | bytes | sql.Composed, params: tuple[Scalar, ...] = ()) -> None: + def execute(self, statement: LiteralString | sql.Composed, params: tuple[Scalar, ...] = ()) -> None: with self.connection() as connection: connection.execute(statement, params or None) diff --git a/tests/e2e/migrations/test_shaped_database.py b/tests/e2e/migrations/test_shaped_database.py index 1c666a4e83d..20c4368ae33 100644 --- a/tests/e2e/migrations/test_shaped_database.py +++ b/tests/e2e/migrations/test_shaped_database.py @@ -1,4 +1,3 @@ -from pathlib import Path from typing import Final import pytest @@ -8,7 +7,6 @@ from .database import Database from .upgrade import assert_history_clean, assert_upgraded, confirm, migration_names, provision SPEND_ROWS: Final = 20_000 -PARTITION_SCRIPT: Final = Path(__file__).parents[3] / "db_scripts" / "partition_spend_logs.sql" pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] @@ -23,13 +21,6 @@ def seed_spend_logs(database: Database, rows: int) -> None: assert database.query('SELECT count(*) FROM "LiteLLM_SpendLogs"') == ((rows,),) -def partition_spend_logs(database: Database) -> None: - database.execute(PARTITION_SCRIPT.read_bytes()) - assert database.query("SELECT relkind::text FROM pg_class WHERE oid = to_regclass('\"LiteLLM_SpendLogs\"')") == ( - ("p",), - ), "The partition script did not leave a partitioned LiteLLM_SpendLogs behind" - - class TestPopulatedDatabaseUpgrade: def test_upgrade_completes_and_preserves_a_populated_spend_log( self, containers: Containers, baseline_image: str, baseline_database: Database @@ -50,29 +41,3 @@ class TestPopulatedDatabaseUpgrade: assert baseline_database.query( 'SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE "startTime" IS NULL OR "endTime" IS NULL' ) == ((0,),), "The upgrade nulled timestamps on existing spend rows" - - def test_upgrade_completes_on_a_partitioned_spend_log( - self, containers: Containers, baseline_image: str, baseline_database: Database - ) -> None: - with containers.using(baseline_image).start(baseline_database) as old: - ready((old,), baseline_database) - key, alias = provision(old) - partition_spend_logs(baseline_database) - seed_spend_logs(baseline_database, SPEND_ROWS) - before: Final = migration_names(baseline_database) - with containers.start(baseline_database) as new: - ready((new,), baseline_database) - assert_upgraded(before, migration_names(baseline_database)) - confirm(new, key, alias) - assert_history_clean(baseline_database) - assert baseline_database.query( - "SELECT relkind::text FROM pg_class WHERE oid = to_regclass('\"LiteLLM_SpendLogs\"')" - ) == (("p",),), "The upgrade replaced the partitioned LiteLLM_SpendLogs with a plain table" - assert baseline_database.query('SELECT count(*) FROM "LiteLLM_SpendLogs"') == ((SPEND_ROWS,),), ( - "The upgrade lost spend rows from the partitioned table" - ) - assert baseline_database.query( - "SELECT count(*) FROM pg_indexes WHERE tablename IN ('LiteLLM_SpendLogs', 'LiteLLM_SpendLogs_pdefault') " - "AND indexdef LIKE %s", - ("%(litellm_call_id)",), - ) == ((2,),), "The litellm_call_id index was not built on the partitioned parent and propagated to its partition" diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index e0923c9d055..92b3c1c993a 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -1,6 +1,6 @@ import os -import socket import signal +import socket import subprocess import sys import time @@ -12,7 +12,6 @@ from typing import Final import httpx import psutil - from integration._support.client import Gateway @@ -46,7 +45,15 @@ def stop_root_process(process: subprocess.Popen[bytes]) -> bool: @contextmanager -def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], *, config: Path | None = None, remove_environment: tuple[str, ...] = ()) -> Iterator[Gateway]: +def owned_proxy( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, + remove_environment: tuple[str, ...] = (), + use_prisma_db_push: bool = True, +) -> Iterator[Gateway]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) port: Final = reserve.getsockname()[1] @@ -61,8 +68,8 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) with (output / f"owned-proxy-{uuid.uuid4().hex}.log").open("w") as log: - process: Final = subprocess.Popen( - [ + argv: Final = ( + ( sys.executable, "-m", "integration._support.proxy", @@ -74,9 +81,12 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], str(port), "--num_workers", "1", - "--use_prisma_db_push", - "--enforce_prisma_migration_check", - ], + ) + + (("--use_prisma_db_push",) if use_prisma_db_push else ()) + + ("--enforce_prisma_migration_check",) + ) + process: Final = subprocess.Popen( + argv, cwd=root, env=environment, stdout=log, diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index e1b5940935f..500a9eb53a3 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -86,6 +86,9 @@ "other.database.partitions.lock_wait_outlives_transaction_default", "other.database.partitions.repeat_preserves_rows" ], + "tests/integration/database/test_partitioned_spend_logs_upgrade.py::test_upgrade_builds_the_call_id_index_on_a_partitioned_spend_log": [ + "other.database.partitions.upgrade_builds_call_id_index_on_partitioned_parent" + ], "tests/integration/database/test_reader_writer_regeneration.py::test_key_regeneration_uses_writer_with_a_real_readonly_reader": [ "other.database.regeneration.writer_updates_dependent_grants" ], diff --git a/tests/integration/database/test_partitioned_spend_logs_upgrade.py b/tests/integration/database/test_partitioned_spend_logs_upgrade.py new file mode 100644 index 00000000000..9e336e79dcd --- /dev/null +++ b/tests/integration/database/test_partitioned_spend_logs_upgrade.py @@ -0,0 +1,98 @@ +import os +import shutil +import subprocess +import sys +import uuid +from pathlib import Path +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from psycopg import sql + +ROOT: Final = Path(__file__).resolve().parents[3] +EXTRAS: Final = ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" +PARTITION_SCRIPT: Final = ROOT / "db_scripts" / "partition_spend_logs.sql" +INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index" +SEED_ROWS: Final = 500 + + +@pytest.mark.timeout(180) +@pytest.mark.covers("other.database.partitions.upgrade_builds_call_id_index_on_partitioned_parent") +def test_upgrade_builds_the_call_id_index_on_a_partitioned_spend_log(gateway: Gateway, tmp_path: Path) -> None: + database: Final = f"integration_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + parsed: Final = urlsplit(url) + database_url: Final = urlunsplit(parsed._replace(path=f"/{database}")) + scoped_url: Final = urlunsplit( + parsed._replace(path=f"/{database}", query=urlencode({**dict(parse_qsl(parsed.query)), "schema": "upgraded"})) + ) + project: Final = tmp_path / "prisma" + staged_migrations: Final = project / "migrations" + staged_migrations.mkdir(parents=True) + shutil.copy2(EXTRAS / "schema.prisma", project / "schema.prisma") + shutil.copy2(EXTRAS / "migrations" / "migration_lock.toml", staged_migrations / "migration_lock.toml") + earlier: Final = tuple( + sorted( + source for source in (EXTRAS / "migrations").iterdir() if source.is_dir() and source.name < "20260831120001" + ) + ) + for source in earlier: + shutil.copytree(source, staged_migrations / source.name) + prisma_bin: Final = Path(sys.executable).with_name("prisma") + assert prisma_bin.exists(), f"Prisma CLI missing next to the test interpreter {sys.executable}" + with psycopg.connect(url, autocommit=True) as admin: + admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database))) + try: + deploy: Final = subprocess.run( + [str(prisma_bin), "migrate", "deploy", "--schema", str(project / "schema.prisma")], + capture_output=True, + text=True, + env={**os.environ, "DATABASE_URL": scoped_url}, + ) + assert deploy.returncode == 0, f"prisma migrate deploy failed before the upgrade: {deploy.stderr}" + with psycopg.connect(database_url, autocommit=True) as setup: + setup.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier("upgraded"))) + setup.execute(PARTITION_SCRIPT.read_bytes()) + assert setup.execute( + "SELECT c.relkind::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = 'upgraded' AND c.relname = 'LiteLLM_SpendLogs'" + ).fetchone() == ("p",), "The partition script did not leave a partitioned LiteLLM_SpendLogs behind" + setup.execute( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, "startTime", "endTime") ' + "SELECT 'upgrade-shape-' || g, 'acompletion', now() - (g || ' seconds')::interval, " + "now() - (g || ' seconds')::interval FROM generate_series(1, %s) AS g", + (SEED_ROWS,), + ) + assert setup.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (SEED_ROWS,), ( + "The seeded spend rows did not land on the partitioned table" + ) + with owned_proxy( + gateway, tmp_path, {"DATABASE_URL": scoped_url}, use_prisma_db_push=False + ) as candidate: + assert candidate.client.get("/health/readiness").status_code == 200, ( + "The upgraded proxy did not become ready against the partitioned database" + ) + assert setup.execute( + "SELECT c.relkind::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = 'upgraded' AND c.relname = 'LiteLLM_SpendLogs'" + ).fetchone() == ("p",), "The upgrade replaced the partitioned LiteLLM_SpendLogs with a plain table" + assert setup.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (SEED_ROWS,), ( + "The upgrade lost spend rows from the partitioned table" + ) + assert setup.execute( + "SELECT count(*) FROM pg_indexes WHERE schemaname = 'upgraded' " + "AND tablename IN ('LiteLLM_SpendLogs', 'LiteLLM_SpendLogs_pdefault') " + "AND indexdef LIKE '%(litellm_call_id)'" + ).fetchone() == (2,), ( + "The litellm_call_id index was not built on the partitioned parent and propagated to its partition" + ) + assert setup.execute( + "SELECT finished_at IS NOT NULL FROM _prisma_migrations WHERE migration_name = %s", + (INDEX_MIGRATION,), + ).fetchall() == [(True,)], "The upgrade did not record the litellm_call_id index migration as applied" + finally: + admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))