test(database): move the partitioned SpendLogs upgrade repro to tests/integration

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-22 18:47:46 +00:00
parent 85a551476b
commit 4b25781248
6 changed files with 121 additions and 45 deletions

View file

@ -14,7 +14,7 @@ SUITES: Final = {
"recovery": (("test_recovery.py",), 15),
"legacy": (("test_legacy.py", "test_pooling.py"), 11),
"upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
"shaped": (("test_shaped_database.py",), 2),
"shaped": (("test_shaped_database.py",), 1),
}

View file

@ -42,7 +42,7 @@ class Database:
connection.execute("SET statement_timeout = '15s'")
yield connection
def execute(self, statement: LiteralString | bytes | sql.Composed, params: tuple[Scalar, ...] = ()) -> None:
def execute(self, statement: LiteralString | sql.Composed, params: tuple[Scalar, ...] = ()) -> None:
with self.connection() as connection:
connection.execute(statement, params or None)

View file

@ -1,4 +1,3 @@
from pathlib import Path
from typing import Final
import pytest
@ -8,7 +7,6 @@ from .database import Database
from .upgrade import assert_history_clean, assert_upgraded, confirm, migration_names, provision
SPEND_ROWS: Final = 20_000
PARTITION_SCRIPT: Final = Path(__file__).parents[3] / "db_scripts" / "partition_spend_logs.sql"
pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
@ -23,13 +21,6 @@ def seed_spend_logs(database: Database, rows: int) -> None:
assert database.query('SELECT count(*) FROM "LiteLLM_SpendLogs"') == ((rows,),)
def partition_spend_logs(database: Database) -> None:
database.execute(PARTITION_SCRIPT.read_bytes())
assert database.query("SELECT relkind::text FROM pg_class WHERE oid = to_regclass('\"LiteLLM_SpendLogs\"')") == (
("p",),
), "The partition script did not leave a partitioned LiteLLM_SpendLogs behind"
class TestPopulatedDatabaseUpgrade:
def test_upgrade_completes_and_preserves_a_populated_spend_log(
self, containers: Containers, baseline_image: str, baseline_database: Database
@ -50,29 +41,3 @@ class TestPopulatedDatabaseUpgrade:
assert baseline_database.query(
'SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE "startTime" IS NULL OR "endTime" IS NULL'
) == ((0,),), "The upgrade nulled timestamps on existing spend rows"
def test_upgrade_completes_on_a_partitioned_spend_log(
self, containers: Containers, baseline_image: str, baseline_database: Database
) -> None:
with containers.using(baseline_image).start(baseline_database) as old:
ready((old,), baseline_database)
key, alias = provision(old)
partition_spend_logs(baseline_database)
seed_spend_logs(baseline_database, SPEND_ROWS)
before: Final = migration_names(baseline_database)
with containers.start(baseline_database) as new:
ready((new,), baseline_database)
assert_upgraded(before, migration_names(baseline_database))
confirm(new, key, alias)
assert_history_clean(baseline_database)
assert baseline_database.query(
"SELECT relkind::text FROM pg_class WHERE oid = to_regclass('\"LiteLLM_SpendLogs\"')"
) == (("p",),), "The upgrade replaced the partitioned LiteLLM_SpendLogs with a plain table"
assert baseline_database.query('SELECT count(*) FROM "LiteLLM_SpendLogs"') == ((SPEND_ROWS,),), (
"The upgrade lost spend rows from the partitioned table"
)
assert baseline_database.query(
"SELECT count(*) FROM pg_indexes WHERE tablename IN ('LiteLLM_SpendLogs', 'LiteLLM_SpendLogs_pdefault') "
"AND indexdef LIKE %s",
("%(litellm_call_id)",),
) == ((2,),), "The litellm_call_id index was not built on the partitioned parent and propagated to its partition"

View file

@ -1,6 +1,6 @@
import os
import socket
import signal
import socket
import subprocess
import sys
import time
@ -12,7 +12,6 @@ from typing import Final
import httpx
import psutil
from integration._support.client import Gateway
@ -46,7 +45,15 @@ def stop_root_process(process: subprocess.Popen[bytes]) -> bool:
@contextmanager
def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], *, config: Path | None = None, remove_environment: tuple[str, ...] = ()) -> Iterator[Gateway]:
def owned_proxy(
gateway: Gateway,
directory: Path,
overrides: Mapping[str, str],
*,
config: Path | None = None,
remove_environment: tuple[str, ...] = (),
use_prisma_db_push: bool = True,
) -> Iterator[Gateway]:
with socket.socket() as reserve:
reserve.bind(("127.0.0.1", 0))
port: Final = reserve.getsockname()[1]
@ -61,8 +68,8 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str],
output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory)))
output.mkdir(parents=True, exist_ok=True)
with (output / f"owned-proxy-{uuid.uuid4().hex}.log").open("w") as log:
process: Final = subprocess.Popen(
[
argv: Final = (
(
sys.executable,
"-m",
"integration._support.proxy",
@ -74,9 +81,12 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str],
str(port),
"--num_workers",
"1",
"--use_prisma_db_push",
"--enforce_prisma_migration_check",
],
)
+ (("--use_prisma_db_push",) if use_prisma_db_push else ())
+ ("--enforce_prisma_migration_check",)
)
process: Final = subprocess.Popen(
argv,
cwd=root,
env=environment,
stdout=log,

View file

@ -86,6 +86,9 @@
"other.database.partitions.lock_wait_outlives_transaction_default",
"other.database.partitions.repeat_preserves_rows"
],
"tests/integration/database/test_partitioned_spend_logs_upgrade.py::test_upgrade_builds_the_call_id_index_on_a_partitioned_spend_log": [
"other.database.partitions.upgrade_builds_call_id_index_on_partitioned_parent"
],
"tests/integration/database/test_reader_writer_regeneration.py::test_key_regeneration_uses_writer_with_a_real_readonly_reader": [
"other.database.regeneration.writer_updates_dependent_grants"
],

View file

@ -0,0 +1,98 @@
import os
import shutil
import subprocess
import sys
import uuid
from pathlib import Path
from typing import Final
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import psycopg
import pytest
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from psycopg import sql
ROOT: Final = Path(__file__).resolve().parents[3]
EXTRAS: Final = ROOT / "litellm-proxy-extras" / "litellm_proxy_extras"
PARTITION_SCRIPT: Final = ROOT / "db_scripts" / "partition_spend_logs.sql"
INDEX_MIGRATION: Final = "20260831120001_spend_logs_litellm_call_id_index"
SEED_ROWS: Final = 500
@pytest.mark.timeout(180)
@pytest.mark.covers("other.database.partitions.upgrade_builds_call_id_index_on_partitioned_parent")
def test_upgrade_builds_the_call_id_index_on_a_partitioned_spend_log(gateway: Gateway, tmp_path: Path) -> None:
database: Final = f"integration_{uuid.uuid4().hex}"
url: Final = os.environ["DATABASE_URL"]
parsed: Final = urlsplit(url)
database_url: Final = urlunsplit(parsed._replace(path=f"/{database}"))
scoped_url: Final = urlunsplit(
parsed._replace(path=f"/{database}", query=urlencode({**dict(parse_qsl(parsed.query)), "schema": "upgraded"}))
)
project: Final = tmp_path / "prisma"
staged_migrations: Final = project / "migrations"
staged_migrations.mkdir(parents=True)
shutil.copy2(EXTRAS / "schema.prisma", project / "schema.prisma")
shutil.copy2(EXTRAS / "migrations" / "migration_lock.toml", staged_migrations / "migration_lock.toml")
earlier: Final = tuple(
sorted(
source for source in (EXTRAS / "migrations").iterdir() if source.is_dir() and source.name < "20260831120001"
)
)
for source in earlier:
shutil.copytree(source, staged_migrations / source.name)
prisma_bin: Final = Path(sys.executable).with_name("prisma")
assert prisma_bin.exists(), f"Prisma CLI missing next to the test interpreter {sys.executable}"
with psycopg.connect(url, autocommit=True) as admin:
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
try:
deploy: Final = subprocess.run(
[str(prisma_bin), "migrate", "deploy", "--schema", str(project / "schema.prisma")],
capture_output=True,
text=True,
env={**os.environ, "DATABASE_URL": scoped_url},
)
assert deploy.returncode == 0, f"prisma migrate deploy failed before the upgrade: {deploy.stderr}"
with psycopg.connect(database_url, autocommit=True) as setup:
setup.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier("upgraded")))
setup.execute(PARTITION_SCRIPT.read_bytes())
assert setup.execute(
"SELECT c.relkind::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
"WHERE n.nspname = 'upgraded' AND c.relname = 'LiteLLM_SpendLogs'"
).fetchone() == ("p",), "The partition script did not leave a partitioned LiteLLM_SpendLogs behind"
setup.execute(
'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, "startTime", "endTime") '
"SELECT 'upgrade-shape-' || g, 'acompletion', now() - (g || ' seconds')::interval, "
"now() - (g || ' seconds')::interval FROM generate_series(1, %s) AS g",
(SEED_ROWS,),
)
assert setup.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (SEED_ROWS,), (
"The seeded spend rows did not land on the partitioned table"
)
with owned_proxy(
gateway, tmp_path, {"DATABASE_URL": scoped_url}, use_prisma_db_push=False
) as candidate:
assert candidate.client.get("/health/readiness").status_code == 200, (
"The upgraded proxy did not become ready against the partitioned database"
)
assert setup.execute(
"SELECT c.relkind::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
"WHERE n.nspname = 'upgraded' AND c.relname = 'LiteLLM_SpendLogs'"
).fetchone() == ("p",), "The upgrade replaced the partitioned LiteLLM_SpendLogs with a plain table"
assert setup.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (SEED_ROWS,), (
"The upgrade lost spend rows from the partitioned table"
)
assert setup.execute(
"SELECT count(*) FROM pg_indexes WHERE schemaname = 'upgraded' "
"AND tablename IN ('LiteLLM_SpendLogs', 'LiteLLM_SpendLogs_pdefault') "
"AND indexdef LIKE '%(litellm_call_id)'"
).fetchone() == (2,), (
"The litellm_call_id index was not built on the partitioned parent and propagated to its partition"
)
assert setup.execute(
"SELECT finished_at IS NOT NULL FROM _prisma_migrations WHERE migration_name = %s",
(INDEX_MIGRATION,),
).fetchall() == [(True,)], "The upgrade did not record the litellm_call_id index migration as applied"
finally:
admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))