mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
refactor(migrations): move LiteLLM_SpendLogs migrations into spend_logs/
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
549e6d4f46
commit
eefa954c47
26 changed files with 199 additions and 48 deletions
3
.github/workflows/test-linting.yml
vendored
3
.github/workflows/test-linting.yml
vendored
|
|
@ -621,6 +621,9 @@ jobs:
|
|||
- name: check_migrations_lock_timeout
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
|
||||
|
||||
- name: check_spend_logs_folder
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_spend_logs_folder.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
|
|
|
|||
|
|
@ -251,8 +251,10 @@ def create_migration(
|
|||
import testing.postgresql
|
||||
|
||||
try:
|
||||
migrations_dir = (
|
||||
root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
|
||||
extras_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras"
|
||||
migration_sources = (
|
||||
extras_dir / "migrations",
|
||||
extras_dir / "spend_logs" / "migrations",
|
||||
)
|
||||
schema_path = root_dir / "schema.prisma"
|
||||
|
||||
|
|
@ -267,7 +269,8 @@ def create_migration(
|
|||
# Copy existing migrations to temp directory
|
||||
if temp_migrations_dir.exists():
|
||||
shutil.rmtree(temp_migrations_dir)
|
||||
shutil.copytree(migrations_dir, temp_migrations_dir)
|
||||
for source in migration_sources:
|
||||
shutil.copytree(source, temp_migrations_dir, dirs_exist_ok=True)
|
||||
|
||||
# Apply existing migrations to temp database
|
||||
os.environ["DATABASE_URL"] = db_url
|
||||
|
|
@ -317,7 +320,13 @@ def create_migration(
|
|||
# Generate timestamp and create migration directory
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
migration_name = migration_name or "unnamed_migration"
|
||||
migration_dir = migrations_dir / f"{timestamp}_{migration_name}"
|
||||
# LiteLLM_SpendLogs migrations live in spend_logs/ so that folder can carry its own code owner
|
||||
destination = (
|
||||
migration_sources[1]
|
||||
if "LiteLLM_SpendLogs" in diff_sql
|
||||
else migration_sources[0]
|
||||
)
|
||||
migration_dir = destination / f"{timestamp}_{migration_name}"
|
||||
migration_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write the SQL to migration.sql
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import hashlib
|
||||
import re
|
||||
import subprocess
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
|
@ -10,6 +10,7 @@ from uuid import uuid4
|
|||
from litellm_proxy_extras import prisma_toolchain
|
||||
from litellm_proxy_extras._logging import logger
|
||||
from litellm_proxy_extras.migration_lock import MigrationCoordinator
|
||||
from litellm_proxy_extras.migration_sources import PACKAGE_DIR, shipped_migrations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import psycopg
|
||||
|
|
@ -80,10 +81,15 @@ def recover_completed_migration(coordinator: MigrationCoordinator, schema: str,
|
|||
|
||||
|
||||
def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
|
||||
return tuple(
|
||||
(path.parent.name, hashlib.sha256(path.read_bytes()).hexdigest())
|
||||
for path in sorted((directory / "migrations").glob("*/migration.sql"))
|
||||
)
|
||||
return _checksums(sorted((directory / "migrations").glob("*/migration.sql")))
|
||||
|
||||
|
||||
def shipped_migration_files() -> tuple[tuple[str, str], ...]:
|
||||
return _checksums(migration / "migration.sql" for migration in shipped_migrations())
|
||||
|
||||
|
||||
def _checksums(scripts: Iterable[Path]) -> tuple[tuple[str, str], ...]:
|
||||
return tuple((path.parent.name, hashlib.sha256(path.read_bytes()).hexdigest()) for path in scripts)
|
||||
|
||||
|
||||
def baseline_current_schema(
|
||||
|
|
@ -95,12 +101,11 @@ def baseline_current_schema(
|
|||
) -> None:
|
||||
from psycopg import sql
|
||||
|
||||
packaged_dir: Final = Path(__file__).parent
|
||||
migrations: Final = migration_files(migrations_dir)
|
||||
if (
|
||||
not migrations
|
||||
or migrations != migration_files(packaged_dir)
|
||||
or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
|
||||
or migrations != shipped_migration_files()
|
||||
or (migrations_dir / "schema.prisma").read_bytes() != (PACKAGE_DIR / "schema.prisma").read_bytes()
|
||||
):
|
||||
raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
import shutil
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
PACKAGE_DIR: Final = Path(__file__).parent
|
||||
MIGRATION_SOURCES: Final = (PACKAGE_DIR / "migrations", PACKAGE_DIR / "spend_logs" / "migrations")
|
||||
|
||||
|
||||
def shipped_migrations() -> tuple[Path, ...]:
|
||||
"""Every packaged migration directory, in the name order `prisma migrate deploy` applies them.
|
||||
|
||||
Migrations that touch LiteLLM_SpendLogs live under `spend_logs/migrations/` so that path can
|
||||
carry its own code owner; Prisma only reads one flat directory, so `materialize_prisma_dir`
|
||||
merges the sources back together before any Prisma command runs."""
|
||||
candidates: Final = chain.from_iterable(source.iterdir() for source in MIGRATION_SOURCES)
|
||||
return tuple(sorted((path for path in candidates if (path / "migration.sql").is_file()), key=lambda p: p.name))
|
||||
|
||||
|
||||
def materialize_prisma_dir(target: Path) -> Path:
|
||||
"""Write the flat Prisma layout (schema.prisma plus one migrations/ directory) into `target`."""
|
||||
migrations: Final = target / "migrations"
|
||||
migrations.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(PACKAGE_DIR / "schema.prisma", target / "schema.prisma")
|
||||
shutil.copy2(PACKAGE_DIR / "migrations" / "migration_lock.toml", migrations / "migration_lock.toml")
|
||||
for migration in shipped_migrations():
|
||||
shutil.copytree(migration, migrations / migration.name, dirs_exist_ok=True)
|
||||
return target
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
model LiteLLM_SpendLogs {
|
||||
request_id String @id
|
||||
call_type String
|
||||
api_key String @default ("") // Hashed API Token. Not the actual Virtual Key. Equivalent to 'token' column in LiteLLM_VerificationToken
|
||||
spend Float @default(0.0)
|
||||
total_tokens Int @default(0)
|
||||
prompt_tokens Int @default(0)
|
||||
completion_tokens Int @default(0)
|
||||
startTime DateTime // Assuming start_time is a DateTime field
|
||||
endTime DateTime // Assuming end_time is a DateTime field
|
||||
request_duration_ms Int?
|
||||
completionStartTime DateTime? // Assuming completionStartTime is a DateTime field
|
||||
model String @default("")
|
||||
model_id String? @default("") // the model id stored in proxy model db
|
||||
model_group String? @default("") // public model_name / model_group
|
||||
custom_llm_provider String? @default("") // litellm used custom_llm_provider
|
||||
api_base String? @default("")
|
||||
user String? @default("")
|
||||
metadata Json? @default("{}") // project_id stored here
|
||||
cache_hit String? @default("")
|
||||
cache_key String? @default("")
|
||||
request_tags Json? @default("[]")
|
||||
team_id String?
|
||||
organization_id String?
|
||||
end_user String?
|
||||
requester_ip_address String?
|
||||
messages Json? @default("{}")
|
||||
response Json? @default("{}")
|
||||
session_id String?
|
||||
status String?
|
||||
mcp_namespaced_tool_name String?
|
||||
billing_agent_id String?
|
||||
agent_id String?
|
||||
proxy_server_request Json? @default("{}")
|
||||
litellm_call_id String?
|
||||
created_at DateTime @default(now()) @map("created_at")
|
||||
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
|
||||
@@index([startTime])
|
||||
@@index([startTime, request_id])
|
||||
@@index([end_user])
|
||||
@@index([session_id])
|
||||
@@index([litellm_call_id])
|
||||
@@index([api_key, startTime])
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import functools
|
||||
import glob
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
|
|
@ -28,7 +28,8 @@ from litellm_proxy_extras.replica_identity import (
|
|||
REPLICA_IDENTITY_FULL_ENV_VAR,
|
||||
apply_replica_identity_full,
|
||||
)
|
||||
from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes, filter_request_log_index_diff
|
||||
from litellm_proxy_extras.migration_sources import materialize_prisma_dir
|
||||
from litellm_proxy_extras.spend_logs.request_log_indexes import ensure_request_log_indexes, filter_request_log_index_diff
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import psycopg
|
||||
|
|
@ -224,6 +225,11 @@ def _redact_command_error(error: subprocess.CalledProcessError) -> str:
|
|||
return str(subprocess.CalledProcessError(error.returncode, redacted_command))
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _default_prisma_dir() -> str:
|
||||
return str(materialize_prisma_dir(Path(tempfile.mkdtemp(prefix="litellm_prisma_"))))
|
||||
|
||||
|
||||
def _get_prisma_command() -> str:
|
||||
"""Get the Prisma command to use, bypassing Python wrapper in offline mode."""
|
||||
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
|
||||
|
|
@ -255,29 +261,14 @@ class ProxyExtrasDBManager:
|
|||
@staticmethod
|
||||
def _get_prisma_dir() -> str:
|
||||
"""
|
||||
Get the path to the migrations directory
|
||||
The flat Prisma directory (schema.prisma plus migrations/) merged from every migration source.
|
||||
|
||||
Set os.environ["LITELLM_MIGRATION_DIR"] to a custom migrations directory, to support baselining db in read-only fs.
|
||||
Set os.environ["LITELLM_MIGRATION_DIR"] to choose where it is written, e.g. a mounted volume on a read-only fs.
|
||||
"""
|
||||
custom_migrations_dir = os.getenv("LITELLM_MIGRATION_DIR")
|
||||
pkg_migrations_dir = os.path.dirname(__file__)
|
||||
custom_migrations_dir: Final = os.getenv("LITELLM_MIGRATION_DIR")
|
||||
if custom_migrations_dir:
|
||||
# If migrations_dir exists, copy contents
|
||||
if os.path.exists(custom_migrations_dir):
|
||||
# Copy contents instead of directory itself
|
||||
for item in os.listdir(pkg_migrations_dir):
|
||||
src_path = os.path.join(pkg_migrations_dir, item)
|
||||
dst_path = os.path.join(custom_migrations_dir, item)
|
||||
if os.path.isdir(src_path):
|
||||
shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(src_path, dst_path)
|
||||
else:
|
||||
# If directory doesn't exist, create it and copy everything
|
||||
shutil.copytree(pkg_migrations_dir, custom_migrations_dir)
|
||||
return custom_migrations_dir
|
||||
|
||||
return pkg_migrations_dir
|
||||
return str(materialize_prisma_dir(Path(custom_migrations_dir)))
|
||||
return _default_prisma_dir()
|
||||
|
||||
@staticmethod
|
||||
def _create_baseline_migration(schema_path: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ from pathlib import Path
|
|||
from typing import Final
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
MIGRATIONS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
|
||||
EXTRAS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras"
|
||||
MIGRATION_SOURCES: Final = (EXTRAS_DIR / "migrations", EXTRAS_DIR / "spend_logs" / "migrations")
|
||||
GRANDFATHERED_THROUGH: Final = "20261009000000_add_credential_display_name"
|
||||
LOCK_TIMEOUT_LINE: Final = "SET LOCAL lock_timeout = '10s';"
|
||||
LOCK_TIMEOUT_RE: Final = re.compile(
|
||||
|
|
@ -41,17 +42,19 @@ def scan_migration(directory: Path) -> tuple[str, ...]:
|
|||
)
|
||||
|
||||
|
||||
def scan_migrations(migrations_dir: Path) -> tuple[str, ...]:
|
||||
directories: Final = tuple(sorted(path for path in migrations_dir.iterdir() if (path / "migration.sql").is_file()))
|
||||
def scan_migrations(migration_sources: tuple[Path, ...]) -> tuple[str, ...]:
|
||||
candidates: Final = chain.from_iterable(source.iterdir() for source in migration_sources)
|
||||
directories: Final = tuple(sorted(path for path in candidates if (path / "migration.sql").is_file()))
|
||||
return tuple(chain.from_iterable(scan_migration(directory) for directory in directories))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not MIGRATIONS_DIR.is_dir():
|
||||
sys.stderr.write(f"migrations directory not found: {MIGRATIONS_DIR}\n")
|
||||
missing: Final = tuple(source for source in MIGRATION_SOURCES if not source.is_dir())
|
||||
if missing:
|
||||
sys.stderr.write(f"migrations directory not found: {missing[0]}\n")
|
||||
return 2
|
||||
|
||||
violations: Final = scan_migrations(MIGRATIONS_DIR)
|
||||
violations: Final = scan_migrations(MIGRATION_SOURCES)
|
||||
for violation in violations:
|
||||
sys.stdout.write(f"{violation}\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ INDEX` blocks writes to the table until the build finishes, and `CREATE INDEX
|
|||
CONCURRENTLY` is refused by Postgres on a partitioned parent, which LiteLLM_SpendLogs
|
||||
is wherever the operator ran db_scripts/partition_spend_logs.sql. The migration job builds
|
||||
those indexes after `migrate deploy`, concurrently and per partition, from the list in
|
||||
litellm_proxy_extras/request_log_indexes.py, so that list is where a new one goes.
|
||||
litellm_proxy_extras/spend_logs/request_log_indexes.py, so that list is where a new one goes.
|
||||
|
||||
Flagged, per statement, by its leading keyword:
|
||||
|
||||
|
|
@ -118,12 +118,14 @@ from __future__ import annotations
|
|||
import re
|
||||
import sys
|
||||
from collections.abc import Iterator, Mapping
|
||||
from itertools import chain
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
MIGRATIONS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
|
||||
EXTRAS_DIR = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras"
|
||||
MIGRATION_SOURCES = (EXTRAS_DIR / "migrations", EXTRAS_DIR / "spend_logs" / "migrations")
|
||||
|
||||
GRANDFATHERED = frozenset(
|
||||
{
|
||||
|
|
@ -220,7 +222,7 @@ statement with the bound spelled out:
|
|||
|
||||
An index on a request-log table is not a migration, and no marker exempts one. Declare it with `@@index` in
|
||||
schema.prisma and add it to REQUEST_LOG_INDEXES in
|
||||
litellm_proxy_extras/request_log_indexes.py under the name Prisma derives for it; the
|
||||
litellm_proxy_extras/spend_logs/request_log_indexes.py under the name Prisma derives for it; the
|
||||
migration job builds it after `migrate deploy`, concurrently on a plain table and per
|
||||
partition on a partitioned one, which no single migration statement can do.
|
||||
|
||||
|
|
@ -238,7 +240,8 @@ class Violation:
|
|||
consequence: str = "rewrites existing rows at boot"
|
||||
|
||||
def render(self) -> str:
|
||||
location = f"{MIGRATIONS_DIR.relative_to(REPO_ROOT)}/{self.migration}/migration.sql"
|
||||
source = next((s for s in MIGRATION_SOURCES if (s / self.migration).is_dir()), MIGRATION_SOURCES[0])
|
||||
location = f"{source.relative_to(REPO_ROOT)}/{self.migration}/migration.sql"
|
||||
return f"{location}:{self.line}: {self.keyword} {self.consequence}"
|
||||
|
||||
|
||||
|
|
@ -623,7 +626,7 @@ def consequence_of(found: str) -> str:
|
|||
if found.startswith("CREATE INDEX"):
|
||||
return (
|
||||
"blocks writes until the build finishes, or fails on a partitioned table; "
|
||||
"add it to REQUEST_LOG_INDEXES in litellm_proxy_extras/request_log_indexes.py instead"
|
||||
"add it to REQUEST_LOG_INDEXES in litellm_proxy_extras/spend_logs/request_log_indexes.py instead"
|
||||
)
|
||||
return "rewrites existing rows at boot"
|
||||
|
||||
|
|
@ -1051,12 +1054,18 @@ def stale_grandfathers(found: Mapping[str, tuple[Violation, ...]]) -> tuple[str,
|
|||
return tuple(sorted((*clean, *missing)))
|
||||
|
||||
|
||||
def shipped_migrations(migration_sources: tuple[Path, ...] = MIGRATION_SOURCES) -> tuple[Path, ...]:
|
||||
candidates = chain.from_iterable(source.iterdir() for source in migration_sources)
|
||||
return tuple(sorted((path for path in candidates if (path / "migration.sql").is_file()), key=lambda p: p.name))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not MIGRATIONS_DIR.is_dir():
|
||||
print(f"migrations directory not found: {MIGRATIONS_DIR}", file=sys.stderr)
|
||||
missing = tuple(source for source in MIGRATION_SOURCES if not source.is_dir())
|
||||
if missing:
|
||||
print(f"migrations directory not found: {missing[0]}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
directories = tuple(sorted(path for path in MIGRATIONS_DIR.iterdir() if (path / "migration.sql").is_file()))
|
||||
directories = shipped_migrations()
|
||||
found = {directory.name: scan_migration(directory) for directory in directories}
|
||||
violations = tuple(
|
||||
violation for name, results in found.items() if name not in GRANDFATHERED for violation in results
|
||||
|
|
|
|||
59
tests/code_coverage_tests/check_spend_logs_folder.py
Normal file
59
tests/code_coverage_tests/check_spend_logs_folder.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Keep every LiteLLM_SpendLogs schema and migration change inside litellm_proxy_extras/spend_logs/.
|
||||
|
||||
That folder carries its own entry in .github/CODEOWNERS, so a migration or index on the spend-log
|
||||
table cannot merge without that owner's review. Two things make the folder meaningful: a migration
|
||||
outside it may not mention LiteLLM_SpendLogs, and spend_logs/LiteLLM_SpendLogs.prisma must be a
|
||||
byte-for-byte copy of the `model LiteLLM_SpendLogs` block in schema.prisma (the pinned Prisma CLI
|
||||
cannot split a schema across files, so the copy is how the schema side lands in the owned folder).
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
EXTRAS_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras"
|
||||
SHARED_MIGRATIONS: Final = EXTRAS_DIR / "migrations"
|
||||
SPEND_LOGS_DIR: Final = EXTRAS_DIR / "spend_logs"
|
||||
SCHEMA: Final = REPO_ROOT / "schema.prisma"
|
||||
TABLE_RE: Final = re.compile(r"\bLiteLLM_SpendLogs\b", re.IGNORECASE)
|
||||
MODEL_BLOCK_RE: Final = re.compile(r"^model LiteLLM_SpendLogs \{\n.*?^\}\n", re.MULTILINE | re.DOTALL)
|
||||
|
||||
|
||||
def misplaced_migrations(shared_migrations: Path) -> tuple[str, ...]:
|
||||
scripts: Final = sorted(shared_migrations.glob("*/migration.sql"))
|
||||
return tuple(
|
||||
f"{script.parent.name}/migration.sql mentions LiteLLM_SpendLogs; move the migration to "
|
||||
f"{SPEND_LOGS_DIR.relative_to(REPO_ROOT)}/migrations/"
|
||||
for script in scripts
|
||||
if TABLE_RE.search(script.read_text(encoding="utf-8"))
|
||||
)
|
||||
|
||||
|
||||
def schema_copy_drift(schema: Path, spend_logs_dir: Path) -> tuple[str, ...]:
|
||||
block: Final = MODEL_BLOCK_RE.search(schema.read_text(encoding="utf-8"))
|
||||
copy: Final = spend_logs_dir / "LiteLLM_SpendLogs.prisma"
|
||||
if block is None:
|
||||
return (f"{schema.relative_to(REPO_ROOT)} has no `model LiteLLM_SpendLogs` block",)
|
||||
if not copy.is_file() or copy.read_text(encoding="utf-8") != block.group():
|
||||
return (
|
||||
f"{copy.relative_to(REPO_ROOT)} must equal the `model LiteLLM_SpendLogs` block in "
|
||||
f"{schema.relative_to(REPO_ROOT)}; copy the block over",
|
||||
)
|
||||
return ()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
violations: Final = (*misplaced_migrations(SHARED_MIGRATIONS), *schema_copy_drift(SCHEMA, SPEND_LOGS_DIR))
|
||||
for violation in violations:
|
||||
sys.stdout.write(f"{violation}\n")
|
||||
if violations:
|
||||
return 1
|
||||
sys.stdout.write("Every LiteLLM_SpendLogs change lives in litellm_proxy_extras/spend_logs/.\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Reference in a new issue