diff --git a/backend/alembic/env.py b/backend/alembic/env.py index a63e6a2..c3e95fd 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -16,8 +16,9 @@ from app.db.models import ( # noqa: F401 — import to register models # this is the Alembic Config object config = context.config -# Override sqlalchemy.url from settings -config.set_main_option("sqlalchemy.url", settings.database_url) +# Override sqlalchemy.url from settings (resolved per db_mode — Postgres +# in shared/team mode; Alembic is not used for local SQLite mode). +config.set_main_option("sqlalchemy.url", settings.resolved_database_url) # Interpret the config file for Python logging if config.config_file_name is not None: diff --git a/backend/app/config.py b/backend/app/config.py index 06e8833..c74dcd0 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,5 +1,12 @@ +"""Application settings and database-mode resolution.""" + +from pathlib import Path + from pydantic_settings import BaseSettings +DEFAULT_POSTGRES_URL = "postgresql://smriti:smriti@localhost:5432/smriti" +DEFAULT_LOCAL_DB_PATH = "~/.smriti/smriti.db" + class Settings(BaseSettings): """Application settings loaded from environment variables.""" @@ -8,8 +15,15 @@ class Settings(BaseSettings): app_name: str = "Smriti" debug: bool = False - # Database - database_url: str = "postgresql://smriti:smriti@localhost:5432/smriti" + # Database — see db_mode / resolved_database_url for how these resolve. + # database_url: the raw DATABASE_URL env var. Empty string means "unset". + database_url: str = "" + # smriti_db_mode: the raw SMRITI_DB_MODE env var — "local" | "postgres". + # Empty string means "derive" (see db_mode). + smriti_db_mode: str = "" + # smriti_local_db_path: SQLite file path for local mode. Empty string + # means the default, ~/.smriti/smriti.db. + smriti_local_db_path: str = "" # OpenAI (for extraction service) openai_api_key: str = "" @@ -20,5 +34,35 @@ class Settings(BaseSettings): model_config = {"env_file": ".env", "env_file_encoding": "utf-8"} + @property + def db_mode(self) -> str: + """Resolved database mode: 'local' or 'postgres'. + + An explicit SMRITI_DB_MODE wins. Otherwise, an explicitly-set + Postgres DATABASE_URL preserves Postgres behavior (backward + compatibility for existing setups); an otherwise-unconfigured + environment defaults to local, so a solo user needs no database + server. + """ + mode = self.smriti_db_mode.strip().lower() + if mode in ("local", "postgres"): + return mode + if self.database_url.strip().startswith(("postgresql://", "postgres://")): + return "postgres" + return "local" + + @property + def local_db_path(self) -> Path: + """Absolute path to the local-mode SQLite database file.""" + raw = self.smriti_local_db_path.strip() or DEFAULT_LOCAL_DB_PATH + return Path(raw).expanduser().resolve() + + @property + def resolved_database_url(self) -> str: + """The SQLAlchemy URL the engine should connect to, per db_mode.""" + if self.db_mode == "postgres": + return self.database_url.strip() or DEFAULT_POSTGRES_URL + return f"sqlite:///{self.local_db_path}" + settings = Settings() diff --git a/backend/app/db/database.py b/backend/app/db/database.py index 54ef541..2277bdd 100644 --- a/backend/app/db/database.py +++ b/backend/app/db/database.py @@ -1,11 +1,44 @@ """SQLAlchemy database engine and session management.""" -from sqlalchemy import create_engine +import logging + +from sqlalchemy import create_engine, event from sqlalchemy.orm import DeclarativeBase, sessionmaker from app.config import settings -engine = create_engine(settings.database_url, echo=settings.debug) +logger = logging.getLogger("uvicorn.error") + + +def _build_engine(): + """Create the SQLAlchemy engine for the resolved database URL. + + Local mode runs against a file-backed SQLite database and needs + cross-thread access plus durability/concurrency pragmas. Postgres + mode is unchanged from prior behavior. + """ + url = settings.resolved_database_url + if url.startswith("sqlite"): + sqlite_engine = create_engine( + url, + echo=settings.debug, + connect_args={"check_same_thread": False}, + ) + + @event.listens_for(sqlite_engine, "connect") + def _set_sqlite_pragmas(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA busy_timeout=5000") + cursor.close() + + return sqlite_engine + + return create_engine(url, echo=settings.debug) + + +engine = _build_engine() SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -13,11 +46,41 @@ class Base(DeclarativeBase): pass -import logging -logger = logging.getLogger("uvicorn.error") +_local_db_ready = False + + +def bootstrap_local_db() -> None: + """Ensure the local SQLite database exists with the current schema. + + Idempotent. No-op in Postgres mode — Alembic owns that schema. In + local mode, creates the parent directory (e.g. ~/.smriti) if needed + and runs create_all, which creates any missing tables and skips + existing ones. + + Must run after every ORM model module is imported, so that + Base.metadata is fully populated before create_all. + """ + global _local_db_ready + if _local_db_ready or settings.db_mode != "local": + return + db_path = settings.local_db_path + first_run = not db_path.exists() + db_path.parent.mkdir(parents=True, exist_ok=True) + Base.metadata.create_all(bind=engine) + if first_run: + logger.info("Smriti local SQLite database created at %s", db_path) + _local_db_ready = True + def get_db(): - """Dependency that yields a database session.""" + """Dependency that yields a database session. + + In local mode the SQLite schema is bootstrapped lazily on first use. + Running the bootstrap here — rather than at import or app startup — + keeps the integration test suite, which overrides this dependency, + fully insulated from the real local database file. + """ + bootstrap_local_db() logger.info("get_db: Creating new session...") db = SessionLocal() logger.info("get_db: Session created.") diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 4ba8935..0e4c1d6 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -4,11 +4,11 @@ import uuid from datetime import datetime, timezone from sqlalchemy import DateTime, Float, ForeignKey, Integer, String, Text -from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import Mapped, mapped_column, relationship -from pgvector.sqlalchemy import Vector from app.db.database import Base +from app.db.types import embedding_column, json_column def _utcnow() -> datetime: @@ -73,11 +73,11 @@ class ExtractionResultModel(Base): unique=True, ) summary: Mapped[str] = mapped_column(Text, nullable=False, default="") - decisions: Mapped[dict] = mapped_column(JSONB, default=list) - tasks: Mapped[dict] = mapped_column(JSONB, default=list) - open_questions: Mapped[dict] = mapped_column(JSONB, default=list) - entities: Mapped[dict] = mapped_column(JSONB, default=list) - code_snippets: Mapped[dict] = mapped_column(JSONB, default=list) + decisions: Mapped[dict] = mapped_column(json_column(), default=list) + tasks: Mapped[dict] = mapped_column(json_column(), default=list) + open_questions: Mapped[dict] = mapped_column(json_column(), default=list) + entities: Mapped[dict] = mapped_column(json_column(), default=list) + code_snippets: Mapped[dict] = mapped_column(json_column(), default=list) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) # Relationships @@ -113,12 +113,12 @@ class MemoryItemModel(Base): ) type: Mapped[str] = mapped_column(String(50), nullable=False) content: Mapped[str] = mapped_column(Text, nullable=False) - embedding: Mapped[list[float] | None] = mapped_column(Vector(1536), nullable=True) # OpenAI small model dim + embedding: Mapped[list[float] | None] = mapped_column(embedding_column(1536), nullable=True) # OpenAI small model dim source: Mapped[str] = mapped_column(String(255), nullable=True) confidence: Mapped[float] = mapped_column(Float, default=1.0) importance: Mapped[float] = mapped_column(Float, default=1.0) status: Mapped[str] = mapped_column(String(50), default="active") - metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=_utcnow, onupdate=_utcnow @@ -137,7 +137,7 @@ class RepoModel(Base): user_id: Mapped[uuid.UUID] = mapped_column( UUID(as_uuid=True), nullable=False, index=True ) - metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=_utcnow, onupdate=_utcnow @@ -173,16 +173,16 @@ class CommitModel(Base): # State snapshots summary: Mapped[str] = mapped_column(Text, default="") objective: Mapped[str] = mapped_column(Text, default="") - decisions: Mapped[dict] = mapped_column(JSONB, default=list) - assumptions: Mapped[dict] = mapped_column(JSONB, default=list) - tasks: Mapped[dict] = mapped_column(JSONB, default=list) - open_questions: Mapped[dict] = mapped_column(JSONB, default=list) - entities: Mapped[dict] = mapped_column(JSONB, default=list) - artifacts: Mapped[dict] = mapped_column(JSONB, default=list) - context_blob: Mapped[dict] = mapped_column(JSONB, default=dict) + decisions: Mapped[dict] = mapped_column(json_column(), default=list) + assumptions: Mapped[dict] = mapped_column(json_column(), default=list) + tasks: Mapped[dict] = mapped_column(json_column(), default=list) + open_questions: Mapped[dict] = mapped_column(json_column(), default=list) + entities: Mapped[dict] = mapped_column(json_column(), default=list) + artifacts: Mapped[dict] = mapped_column(json_column(), default=list) + context_blob: Mapped[dict] = mapped_column(json_column(), default=dict) raw_source_text: Mapped[str | None] = mapped_column(Text, nullable=True) - metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) # Relationships @@ -220,7 +220,7 @@ class ChatSession(Base): # ## Active branches section of smriti state without deleting history. # Values: "active" (default, shown), "integrated" (hidden), "abandoned" (hidden). branch_disposition: Mapped[str] = mapped_column(String(20), default="active") - metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=_utcnow, onupdate=_utcnow @@ -258,7 +258,7 @@ class TurnEvent(Base): commit_id: Mapped[uuid.UUID | None] = mapped_column( UUID(as_uuid=True), ForeignKey("commits.id", ondelete="SET NULL"), nullable=True ) - metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) # Relationships diff --git a/backend/app/db/types.py b/backend/app/db/types.py new file mode 100644 index 0000000..7872632 --- /dev/null +++ b/backend/app/db/types.py @@ -0,0 +1,36 @@ +"""Portable SQLAlchemy column types. + +Smriti's ORM models target PostgreSQL in the shared/team mode and SQLite +in the local/solo mode. These helpers render the strong PostgreSQL type +on the ``postgresql`` dialect and a portable equivalent everywhere else +(SQLite), so the same models and the same ``create_all`` work against +both backends with no per-dialect substitution. +""" + +from __future__ import annotations + +from sqlalchemy import JSON +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.types import TypeEngine + +from pgvector.sqlalchemy import Vector + + +def json_column() -> TypeEngine: + """JSON storage column. + + Renders as ``JSONB`` on PostgreSQL (unchanged from prior behavior) + and as generic ``JSON`` (TEXT-backed) on SQLite. + """ + return JSON().with_variant(JSONB(), "postgresql") + + +def embedding_column(dim: int) -> TypeEngine: + """Vector embedding column. + + Renders as a pgvector ``VECTOR`` on PostgreSQL and as generic + ``JSON`` on SQLite. Local mode does not provide vector search; the + column only needs to round-trip so the legacy memories table can be + created on SQLite. + """ + return Vector(dim).with_variant(JSON(), "sqlite") diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index ce8a659..04ee572 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -1,16 +1,17 @@ -"""Integration test configuration — uses SQLite in-memory database. +"""Integration test configuration — uses an in-memory SQLite database. -This conftest overrides the FastAPI app's `get_db` dependency to use -a fresh in-memory SQLite database for each test session, enabling -full API integration tests without requiring PostgreSQL. +Overrides the FastAPI app's `get_db` dependency to use a fresh in-memory +SQLite database for each test, enabling full API integration tests +without requiring PostgreSQL. -JSONB columns are mapped to JSON (TEXT-backed) for SQLite compatibility. +The ORM models use portable column types (see `app/db/types.py`), so +`create_all` works against SQLite natively — no JSONB/VECTOR DDL +substitution is needed here. """ import pytest from fastapi.testclient import TestClient -from sqlalchemy import JSON, create_engine, event -from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool @@ -26,56 +27,14 @@ def db_engine(): poolclass=StaticPool, ) - # Enable foreign keys in SQLite + # Enable foreign keys in SQLite (off by default). @event.listens_for(engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() cursor.execute("PRAGMA foreign_keys=ON") cursor.close() - # Map JSONB → JSON for SQLite (JSONB is Postgres-only) - # We temporarily swap the impl so create_all works on SQLite - _render_original = JSONB().compile - - @event.listens_for(engine, "before_cursor_execute", retval=True) - def receive_before_cursor_execute(conn, cursor, statement, parameters, context, executemany): - # Replace JSONB with JSON in DDL statements for SQLite - if "JSONB" in statement: - statement = statement.replace("JSONB", "JSON") - if "VECTOR" in statement: - statement = statement.replace("VECTOR", "JSON") - return statement, parameters - - # Create tables, mapping JSONB and Vector for SQLite - from sqlalchemy.dialects import sqlite as sqlite_dialect - from sqlalchemy.dialects.sqlite.base import SQLiteTypeCompiler - from pgvector.sqlalchemy import Vector - - original_visit_jsonb = getattr(SQLiteTypeCompiler, "visit_JSONB", None) - original_visit_vector = getattr(SQLiteTypeCompiler, "visit_vector", None) - - SQLiteTypeCompiler.visit_JSONB = lambda self, type_, **kw: "JSON" - SQLiteTypeCompiler.visit_vector = lambda self, type_, **kw: "JSON" - Base.metadata.create_all(bind=engine) - - # Restore - if original_visit_jsonb: - SQLiteTypeCompiler.visit_JSONB = original_visit_jsonb - else: - try: - del SQLiteTypeCompiler.visit_JSONB - except AttributeError: - pass - - if original_visit_vector: - SQLiteTypeCompiler.visit_vector = original_visit_vector - else: - try: - del SQLiteTypeCompiler.visit_vector - except AttributeError: - pass - yield engine Base.metadata.drop_all(bind=engine) diff --git a/backend/tests/integration/test_local_sqlite_smoke.py b/backend/tests/integration/test_local_sqlite_smoke.py new file mode 100644 index 0000000..9df97ba --- /dev/null +++ b/backend/tests/integration/test_local_sqlite_smoke.py @@ -0,0 +1,178 @@ +"""Local-first SQLite mode — smoke and resolution tests. + +Covers: +- db_mode / resolved_database_url resolution (local default, the Postgres + DATABASE_URL backward-compat bridge, explicit overrides, path override) +- portable JSON type renders JSONB on PostgreSQL, JSON on SQLite +- a persistent file-backed SQLite end-to-end flow (space -> checkpoint -> + claim -> current state -> note) whose data survives reopening the engine +""" + +from __future__ import annotations + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, event, select +from sqlalchemy.dialects import postgresql, sqlite +from sqlalchemy.orm import sessionmaker + +from app.config import DEFAULT_POSTGRES_URL, Settings +from app.db.database import Base, get_db +from app.db.models import CommitModel, RepoModel +from app.db.types import json_column +from app.main import app + + +# ── db_mode / URL resolution ───────────────────────────────────────────────── + + +def _settings(**overrides) -> Settings: + """Construct Settings hermetically — explicit kwargs win over ambient env.""" + base = {"database_url": "", "smriti_db_mode": "", "smriti_local_db_path": ""} + base.update(overrides) + return Settings(_env_file=None, **base) + + +def test_db_mode_defaults_to_local_when_unconfigured(): + s = _settings() + assert s.db_mode == "local" + assert s.resolved_database_url.startswith("sqlite:///") + assert s.resolved_database_url.endswith("smriti.db") + + +def test_postgres_database_url_preserves_postgres_mode(): + """An explicitly-set Postgres DATABASE_URL keeps Postgres — backward compat.""" + s = _settings(database_url="postgresql://u:p@dbhost:5432/smriti") + assert s.db_mode == "postgres" + assert s.resolved_database_url == "postgresql://u:p@dbhost:5432/smriti" + + +def test_explicit_db_mode_overrides_database_url(): + """SMRITI_DB_MODE=local wins even if a Postgres DATABASE_URL is present.""" + s = _settings(smriti_db_mode="local", database_url="postgresql://u:p@h:5432/db") + assert s.db_mode == "local" + assert s.resolved_database_url.startswith("sqlite:///") + + s2 = _settings(smriti_db_mode="postgres") + assert s2.db_mode == "postgres" + assert s2.resolved_database_url == DEFAULT_POSTGRES_URL + + +def test_local_db_path_is_overridable(tmp_path): + custom = tmp_path / "custom-smriti.db" + s = _settings(smriti_db_mode="local", smriti_local_db_path=str(custom)) + assert s.db_mode == "local" + assert s.resolved_database_url.startswith("sqlite:///") + assert s.resolved_database_url.endswith("custom-smriti.db") + + +# ── portable types ─────────────────────────────────────────────────────────── + + +def test_portable_json_renders_jsonb_on_postgres_and_json_on_sqlite(): + rendered_pg = json_column().compile(dialect=postgresql.dialect()) + rendered_sqlite = json_column().compile(dialect=sqlite.dialect()) + assert "JSONB" in str(rendered_pg) # no Postgres regression + assert "JSON" in str(rendered_sqlite) # portable on SQLite + + +# ── persistent file-backed SQLite end-to-end ───────────────────────────────── + + +def _sqlite_file_engine(url: str): + engine = create_engine(url, connect_args={"check_same_thread": False}) + + @event.listens_for(engine, "connect") + def _pragmas(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + return engine + + +def test_local_sqlite_persistence(tmp_path): + """A real .db file: full API flow, then data survives an engine reopen.""" + db_file = tmp_path / "smriti.db" + url = f"sqlite:///{db_file}" + + engine = _sqlite_file_engine(url) + Base.metadata.create_all(bind=engine) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + def override_get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + app.dependency_overrides[get_db] = override_get_db + try: + with TestClient(app) as c: + r = c.post("/api/v2/repos", json={"name": "Local Smoke Space"}) + assert r.status_code == 201, r.text + space_id = r.json()["id"] + + r = c.post( + f"/api/v4/chat/spaces/{space_id}/sessions", + json={"title": "smoke", "provider": "openrouter", "model": "mock"}, + ) + assert r.status_code == 201, r.text + session_id = r.json()["id"] + + r = c.post( + "/api/v4/chat/commit", + json={ + "repo_id": space_id, + "session_id": session_id, + "message": "Local smoke checkpoint", + "decisions": ["SQLite local mode works"], + "tasks": [{"text": "verify persistence", "intent_hint": "test"}], + }, + ) + assert r.status_code == 201, r.text + checkpoint_id = r.json()["id"] + + r = c.post( + "/api/v5/claims", + json={ + "space_id": space_id, + "agent": "claude-code", + "scope": "local smoke", + "intent_type": "test", + }, + ) + assert r.status_code == 201, r.text + + r = c.get(f"/api/v5/current/spaces/{space_id}") + assert r.status_code == 200, r.text + current = r.json() + assert current["counts"]["checkpoints"] == 1 + assert current["counts"]["active_claims"] == 1 + + r = c.post( + f"/api/v5/checkpoint/{checkpoint_id}/notes", + json={"text": "local smoke note", "kind": "milestone", "author": "claude-code"}, + ) + assert r.status_code == 201, r.text + finally: + app.dependency_overrides.clear() + engine.dispose() + + # The data must survive a fresh engine opened against the same file. + assert db_file.exists() + engine2 = _sqlite_file_engine(url) + reopened = sessionmaker(autocommit=False, autoflush=False, bind=engine2)() + try: + repos = reopened.scalars(select(RepoModel)).all() + assert [r.name for r in repos] == ["Local Smoke Space"] + + commits = reopened.scalars(select(CommitModel)).all() + assert len(commits) == 1 + # JSON columns round-trip through SQLite. + assert commits[0].decisions == ["SQLite local mode works"] + notes = (commits[0].metadata_ or {}).get("notes") or [] + assert any(n.get("text") == "local smoke note" for n in notes) + finally: + reopened.close() + engine2.dispose()