Add optional local-first SQLite mode (Phase A core)

Smriti can now run against a file-backed SQLite database with no Docker
and no Postgres, removing the operate-a-backend burden for solo builders.
Postgres remains the stronger shared/team mode, unchanged.

- Portable column types (app/db/types.py): JSON renders as JSONB on
  PostgreSQL and generic JSON on SQLite; the pgvector embedding column
  renders as JSON on SQLite. Same models and create_all on both backends.
- Mode resolution (config.py): SMRITI_DB_MODE=local|postgres, defaulting
  to local when unconfigured. An explicitly-set Postgres DATABASE_URL
  preserves Postgres behavior, so existing setups are unaffected. Local
  DB defaults to ~/.smriti/smriti.db; SMRITI_LOCAL_DB_PATH overrides.
- SQLite engine setup (database.py): check_same_thread plus foreign_keys
  / WAL / busy_timeout pragmas; a lazy first-run create_all bootstrap on
  first DB use, which keeps the integration test suite insulated.
- Removed the integration-test JSONB/VECTOR DDL substitution hack — the
  models are genuinely portable now, so conftest needs no type patching.
- Alembic resolves its URL through the same logic (Postgres mode only).
- New persistent file-backed SQLite smoke test, plus mode-resolution and
  per-dialect type-rendering tests.

Local mode uses create_all, not Alembic. No schema changes.
This commit is contained in:
Himanshu Dongre 2026-05-16 22:26:05 +05:30
parent 83e7aa15cb
commit f22056418d
7 changed files with 360 additions and 79 deletions

View file

@ -16,8 +16,9 @@ from app.db.models import ( # noqa: F401 — import to register models
# this is the Alembic Config object
config = context.config
# Override sqlalchemy.url from settings
config.set_main_option("sqlalchemy.url", settings.database_url)
# Override sqlalchemy.url from settings (resolved per db_mode — Postgres
# in shared/team mode; Alembic is not used for local SQLite mode).
config.set_main_option("sqlalchemy.url", settings.resolved_database_url)
# Interpret the config file for Python logging
if config.config_file_name is not None:

View file

@ -1,5 +1,12 @@
"""Application settings and database-mode resolution."""
from pathlib import Path
from pydantic_settings import BaseSettings
DEFAULT_POSTGRES_URL = "postgresql://smriti:smriti@localhost:5432/smriti"
DEFAULT_LOCAL_DB_PATH = "~/.smriti/smriti.db"
class Settings(BaseSettings):
"""Application settings loaded from environment variables."""
@ -8,8 +15,15 @@ class Settings(BaseSettings):
app_name: str = "Smriti"
debug: bool = False
# Database
database_url: str = "postgresql://smriti:smriti@localhost:5432/smriti"
# Database — see db_mode / resolved_database_url for how these resolve.
# database_url: the raw DATABASE_URL env var. Empty string means "unset".
database_url: str = ""
# smriti_db_mode: the raw SMRITI_DB_MODE env var — "local" | "postgres".
# Empty string means "derive" (see db_mode).
smriti_db_mode: str = ""
# smriti_local_db_path: SQLite file path for local mode. Empty string
# means the default, ~/.smriti/smriti.db.
smriti_local_db_path: str = ""
# OpenAI (for extraction service)
openai_api_key: str = ""
@ -20,5 +34,35 @@ class Settings(BaseSettings):
model_config = {"env_file": ".env", "env_file_encoding": "utf-8"}
@property
def db_mode(self) -> str:
"""Resolved database mode: 'local' or 'postgres'.
An explicit SMRITI_DB_MODE wins. Otherwise, an explicitly-set
Postgres DATABASE_URL preserves Postgres behavior (backward
compatibility for existing setups); an otherwise-unconfigured
environment defaults to local, so a solo user needs no database
server.
"""
mode = self.smriti_db_mode.strip().lower()
if mode in ("local", "postgres"):
return mode
if self.database_url.strip().startswith(("postgresql://", "postgres://")):
return "postgres"
return "local"
@property
def local_db_path(self) -> Path:
"""Absolute path to the local-mode SQLite database file."""
raw = self.smriti_local_db_path.strip() or DEFAULT_LOCAL_DB_PATH
return Path(raw).expanduser().resolve()
@property
def resolved_database_url(self) -> str:
"""The SQLAlchemy URL the engine should connect to, per db_mode."""
if self.db_mode == "postgres":
return self.database_url.strip() or DEFAULT_POSTGRES_URL
return f"sqlite:///{self.local_db_path}"
settings = Settings()

View file

@ -1,11 +1,44 @@
"""SQLAlchemy database engine and session management."""
from sqlalchemy import create_engine
import logging
from sqlalchemy import create_engine, event
from sqlalchemy.orm import DeclarativeBase, sessionmaker
from app.config import settings
engine = create_engine(settings.database_url, echo=settings.debug)
logger = logging.getLogger("uvicorn.error")
def _build_engine():
"""Create the SQLAlchemy engine for the resolved database URL.
Local mode runs against a file-backed SQLite database and needs
cross-thread access plus durability/concurrency pragmas. Postgres
mode is unchanged from prior behavior.
"""
url = settings.resolved_database_url
if url.startswith("sqlite"):
sqlite_engine = create_engine(
url,
echo=settings.debug,
connect_args={"check_same_thread": False},
)
@event.listens_for(sqlite_engine, "connect")
def _set_sqlite_pragmas(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=5000")
cursor.close()
return sqlite_engine
return create_engine(url, echo=settings.debug)
engine = _build_engine()
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
@ -13,11 +46,41 @@ class Base(DeclarativeBase):
pass
import logging
logger = logging.getLogger("uvicorn.error")
_local_db_ready = False
def bootstrap_local_db() -> None:
"""Ensure the local SQLite database exists with the current schema.
Idempotent. No-op in Postgres mode Alembic owns that schema. In
local mode, creates the parent directory (e.g. ~/.smriti) if needed
and runs create_all, which creates any missing tables and skips
existing ones.
Must run after every ORM model module is imported, so that
Base.metadata is fully populated before create_all.
"""
global _local_db_ready
if _local_db_ready or settings.db_mode != "local":
return
db_path = settings.local_db_path
first_run = not db_path.exists()
db_path.parent.mkdir(parents=True, exist_ok=True)
Base.metadata.create_all(bind=engine)
if first_run:
logger.info("Smriti local SQLite database created at %s", db_path)
_local_db_ready = True
def get_db():
"""Dependency that yields a database session."""
"""Dependency that yields a database session.
In local mode the SQLite schema is bootstrapped lazily on first use.
Running the bootstrap here rather than at import or app startup
keeps the integration test suite, which overrides this dependency,
fully insulated from the real local database file.
"""
bootstrap_local_db()
logger.info("get_db: Creating new session...")
db = SessionLocal()
logger.info("get_db: Session created.")

View file

@ -4,11 +4,11 @@ import uuid
from datetime import datetime, timezone
from sqlalchemy import DateTime, Float, ForeignKey, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from pgvector.sqlalchemy import Vector
from app.db.database import Base
from app.db.types import embedding_column, json_column
def _utcnow() -> datetime:
@ -73,11 +73,11 @@ class ExtractionResultModel(Base):
unique=True,
)
summary: Mapped[str] = mapped_column(Text, nullable=False, default="")
decisions: Mapped[dict] = mapped_column(JSONB, default=list)
tasks: Mapped[dict] = mapped_column(JSONB, default=list)
open_questions: Mapped[dict] = mapped_column(JSONB, default=list)
entities: Mapped[dict] = mapped_column(JSONB, default=list)
code_snippets: Mapped[dict] = mapped_column(JSONB, default=list)
decisions: Mapped[dict] = mapped_column(json_column(), default=list)
tasks: Mapped[dict] = mapped_column(json_column(), default=list)
open_questions: Mapped[dict] = mapped_column(json_column(), default=list)
entities: Mapped[dict] = mapped_column(json_column(), default=list)
code_snippets: Mapped[dict] = mapped_column(json_column(), default=list)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
# Relationships
@ -113,12 +113,12 @@ class MemoryItemModel(Base):
)
type: Mapped[str] = mapped_column(String(50), nullable=False)
content: Mapped[str] = mapped_column(Text, nullable=False)
embedding: Mapped[list[float] | None] = mapped_column(Vector(1536), nullable=True) # OpenAI small model dim
embedding: Mapped[list[float] | None] = mapped_column(embedding_column(1536), nullable=True) # OpenAI small model dim
source: Mapped[str] = mapped_column(String(255), nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
importance: Mapped[float] = mapped_column(Float, default=1.0)
status: Mapped[str] = mapped_column(String(50), default="active")
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=_utcnow, onupdate=_utcnow
@ -137,7 +137,7 @@ class RepoModel(Base):
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), nullable=False, index=True
)
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=_utcnow, onupdate=_utcnow
@ -173,16 +173,16 @@ class CommitModel(Base):
# State snapshots
summary: Mapped[str] = mapped_column(Text, default="")
objective: Mapped[str] = mapped_column(Text, default="")
decisions: Mapped[dict] = mapped_column(JSONB, default=list)
assumptions: Mapped[dict] = mapped_column(JSONB, default=list)
tasks: Mapped[dict] = mapped_column(JSONB, default=list)
open_questions: Mapped[dict] = mapped_column(JSONB, default=list)
entities: Mapped[dict] = mapped_column(JSONB, default=list)
artifacts: Mapped[dict] = mapped_column(JSONB, default=list)
context_blob: Mapped[dict] = mapped_column(JSONB, default=dict)
decisions: Mapped[dict] = mapped_column(json_column(), default=list)
assumptions: Mapped[dict] = mapped_column(json_column(), default=list)
tasks: Mapped[dict] = mapped_column(json_column(), default=list)
open_questions: Mapped[dict] = mapped_column(json_column(), default=list)
entities: Mapped[dict] = mapped_column(json_column(), default=list)
artifacts: Mapped[dict] = mapped_column(json_column(), default=list)
context_blob: Mapped[dict] = mapped_column(json_column(), default=dict)
raw_source_text: Mapped[str | None] = mapped_column(Text, nullable=True)
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
# Relationships
@ -220,7 +220,7 @@ class ChatSession(Base):
# ## Active branches section of smriti state without deleting history.
# Values: "active" (default, shown), "integrated" (hidden), "abandoned" (hidden).
branch_disposition: Mapped[str] = mapped_column(String(20), default="active")
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=_utcnow, onupdate=_utcnow
@ -258,7 +258,7 @@ class TurnEvent(Base):
commit_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("commits.id", ondelete="SET NULL"), nullable=True
)
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
metadata_: Mapped[dict] = mapped_column("metadata", json_column(), default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
# Relationships

36
backend/app/db/types.py Normal file
View file

@ -0,0 +1,36 @@
"""Portable SQLAlchemy column types.
Smriti's ORM models target PostgreSQL in the shared/team mode and SQLite
in the local/solo mode. These helpers render the strong PostgreSQL type
on the ``postgresql`` dialect and a portable equivalent everywhere else
(SQLite), so the same models and the same ``create_all`` work against
both backends with no per-dialect substitution.
"""
from __future__ import annotations
from sqlalchemy import JSON
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.types import TypeEngine
from pgvector.sqlalchemy import Vector
def json_column() -> TypeEngine:
"""JSON storage column.
Renders as ``JSONB`` on PostgreSQL (unchanged from prior behavior)
and as generic ``JSON`` (TEXT-backed) on SQLite.
"""
return JSON().with_variant(JSONB(), "postgresql")
def embedding_column(dim: int) -> TypeEngine:
"""Vector embedding column.
Renders as a pgvector ``VECTOR`` on PostgreSQL and as generic
``JSON`` on SQLite. Local mode does not provide vector search; the
column only needs to round-trip so the legacy memories table can be
created on SQLite.
"""
return Vector(dim).with_variant(JSON(), "sqlite")

View file

@ -1,16 +1,17 @@
"""Integration test configuration — uses SQLite in-memory database.
"""Integration test configuration — uses an in-memory SQLite database.
This conftest overrides the FastAPI app's `get_db` dependency to use
a fresh in-memory SQLite database for each test session, enabling
full API integration tests without requiring PostgreSQL.
Overrides the FastAPI app's `get_db` dependency to use a fresh in-memory
SQLite database for each test, enabling full API integration tests
without requiring PostgreSQL.
JSONB columns are mapped to JSON (TEXT-backed) for SQLite compatibility.
The ORM models use portable column types (see `app/db/types.py`), so
`create_all` works against SQLite natively no JSONB/VECTOR DDL
substitution is needed here.
"""
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import JSON, create_engine, event
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
@ -26,56 +27,14 @@ def db_engine():
poolclass=StaticPool,
)
# Enable foreign keys in SQLite
# Enable foreign keys in SQLite (off by default).
@event.listens_for(engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
# Map JSONB → JSON for SQLite (JSONB is Postgres-only)
# We temporarily swap the impl so create_all works on SQLite
_render_original = JSONB().compile
@event.listens_for(engine, "before_cursor_execute", retval=True)
def receive_before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
# Replace JSONB with JSON in DDL statements for SQLite
if "JSONB" in statement:
statement = statement.replace("JSONB", "JSON")
if "VECTOR" in statement:
statement = statement.replace("VECTOR", "JSON")
return statement, parameters
# Create tables, mapping JSONB and Vector for SQLite
from sqlalchemy.dialects import sqlite as sqlite_dialect
from sqlalchemy.dialects.sqlite.base import SQLiteTypeCompiler
from pgvector.sqlalchemy import Vector
original_visit_jsonb = getattr(SQLiteTypeCompiler, "visit_JSONB", None)
original_visit_vector = getattr(SQLiteTypeCompiler, "visit_vector", None)
SQLiteTypeCompiler.visit_JSONB = lambda self, type_, **kw: "JSON"
SQLiteTypeCompiler.visit_vector = lambda self, type_, **kw: "JSON"
Base.metadata.create_all(bind=engine)
# Restore
if original_visit_jsonb:
SQLiteTypeCompiler.visit_JSONB = original_visit_jsonb
else:
try:
del SQLiteTypeCompiler.visit_JSONB
except AttributeError:
pass
if original_visit_vector:
SQLiteTypeCompiler.visit_vector = original_visit_vector
else:
try:
del SQLiteTypeCompiler.visit_vector
except AttributeError:
pass
yield engine
Base.metadata.drop_all(bind=engine)

View file

@ -0,0 +1,178 @@
"""Local-first SQLite mode — smoke and resolution tests.
Covers:
- db_mode / resolved_database_url resolution (local default, the Postgres
DATABASE_URL backward-compat bridge, explicit overrides, path override)
- portable JSON type renders JSONB on PostgreSQL, JSON on SQLite
- a persistent file-backed SQLite end-to-end flow (space -> checkpoint ->
claim -> current state -> note) whose data survives reopening the engine
"""
from __future__ import annotations
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, event, select
from sqlalchemy.dialects import postgresql, sqlite
from sqlalchemy.orm import sessionmaker
from app.config import DEFAULT_POSTGRES_URL, Settings
from app.db.database import Base, get_db
from app.db.models import CommitModel, RepoModel
from app.db.types import json_column
from app.main import app
# ── db_mode / URL resolution ─────────────────────────────────────────────────
def _settings(**overrides) -> Settings:
"""Construct Settings hermetically — explicit kwargs win over ambient env."""
base = {"database_url": "", "smriti_db_mode": "", "smriti_local_db_path": ""}
base.update(overrides)
return Settings(_env_file=None, **base)
def test_db_mode_defaults_to_local_when_unconfigured():
s = _settings()
assert s.db_mode == "local"
assert s.resolved_database_url.startswith("sqlite:///")
assert s.resolved_database_url.endswith("smriti.db")
def test_postgres_database_url_preserves_postgres_mode():
"""An explicitly-set Postgres DATABASE_URL keeps Postgres — backward compat."""
s = _settings(database_url="postgresql://u:p@dbhost:5432/smriti")
assert s.db_mode == "postgres"
assert s.resolved_database_url == "postgresql://u:p@dbhost:5432/smriti"
def test_explicit_db_mode_overrides_database_url():
"""SMRITI_DB_MODE=local wins even if a Postgres DATABASE_URL is present."""
s = _settings(smriti_db_mode="local", database_url="postgresql://u:p@h:5432/db")
assert s.db_mode == "local"
assert s.resolved_database_url.startswith("sqlite:///")
s2 = _settings(smriti_db_mode="postgres")
assert s2.db_mode == "postgres"
assert s2.resolved_database_url == DEFAULT_POSTGRES_URL
def test_local_db_path_is_overridable(tmp_path):
custom = tmp_path / "custom-smriti.db"
s = _settings(smriti_db_mode="local", smriti_local_db_path=str(custom))
assert s.db_mode == "local"
assert s.resolved_database_url.startswith("sqlite:///")
assert s.resolved_database_url.endswith("custom-smriti.db")
# ── portable types ───────────────────────────────────────────────────────────
def test_portable_json_renders_jsonb_on_postgres_and_json_on_sqlite():
rendered_pg = json_column().compile(dialect=postgresql.dialect())
rendered_sqlite = json_column().compile(dialect=sqlite.dialect())
assert "JSONB" in str(rendered_pg) # no Postgres regression
assert "JSON" in str(rendered_sqlite) # portable on SQLite
# ── persistent file-backed SQLite end-to-end ─────────────────────────────────
def _sqlite_file_engine(url: str):
engine = create_engine(url, connect_args={"check_same_thread": False})
@event.listens_for(engine, "connect")
def _pragmas(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
return engine
def test_local_sqlite_persistence(tmp_path):
"""A real .db file: full API flow, then data survives an engine reopen."""
db_file = tmp_path / "smriti.db"
url = f"sqlite:///{db_file}"
engine = _sqlite_file_engine(url)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
def override_get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
app.dependency_overrides[get_db] = override_get_db
try:
with TestClient(app) as c:
r = c.post("/api/v2/repos", json={"name": "Local Smoke Space"})
assert r.status_code == 201, r.text
space_id = r.json()["id"]
r = c.post(
f"/api/v4/chat/spaces/{space_id}/sessions",
json={"title": "smoke", "provider": "openrouter", "model": "mock"},
)
assert r.status_code == 201, r.text
session_id = r.json()["id"]
r = c.post(
"/api/v4/chat/commit",
json={
"repo_id": space_id,
"session_id": session_id,
"message": "Local smoke checkpoint",
"decisions": ["SQLite local mode works"],
"tasks": [{"text": "verify persistence", "intent_hint": "test"}],
},
)
assert r.status_code == 201, r.text
checkpoint_id = r.json()["id"]
r = c.post(
"/api/v5/claims",
json={
"space_id": space_id,
"agent": "claude-code",
"scope": "local smoke",
"intent_type": "test",
},
)
assert r.status_code == 201, r.text
r = c.get(f"/api/v5/current/spaces/{space_id}")
assert r.status_code == 200, r.text
current = r.json()
assert current["counts"]["checkpoints"] == 1
assert current["counts"]["active_claims"] == 1
r = c.post(
f"/api/v5/checkpoint/{checkpoint_id}/notes",
json={"text": "local smoke note", "kind": "milestone", "author": "claude-code"},
)
assert r.status_code == 201, r.text
finally:
app.dependency_overrides.clear()
engine.dispose()
# The data must survive a fresh engine opened against the same file.
assert db_file.exists()
engine2 = _sqlite_file_engine(url)
reopened = sessionmaker(autocommit=False, autoflush=False, bind=engine2)()
try:
repos = reopened.scalars(select(RepoModel)).all()
assert [r.name for r in repos] == ["Local Smoke Space"]
commits = reopened.scalars(select(CommitModel)).all()
assert len(commits) == 1
# JSON columns round-trip through SQLite.
assert commits[0].decisions == ["SQLite local mode works"]
notes = (commits[0].metadata_ or {}).get("notes") or []
assert any(n.get("text") == "local smoke note" for n in notes)
finally:
reopened.close()
engine2.dispose()