diff --git a/apps/desktop/scripts/run-tests.mjs b/apps/desktop/scripts/run-tests.mjs index c666926d..fb428e3d 100644 --- a/apps/desktop/scripts/run-tests.mjs +++ b/apps/desktop/scripts/run-tests.mjs @@ -61,6 +61,7 @@ const behaviorTests = [ ".test-dist/tests/claude-memory.test.js", ".test-dist/tests/prompt-memory-extraction.test.js", ".test-dist/tests/familiaros-memory-search.test.js", + ".test-dist/tests/knowledge-store-core-seam.test.js", ".test-dist/tests/knowledge-store.test.js", ".test-dist/tests/control-center-service-barrels.test.js", ".test-dist/tests/mcp-toolkit-catalog.test.js", diff --git a/apps/desktop/src/check-packaging-contract.ts b/apps/desktop/src/check-packaging-contract.ts index bb694bb6..a7870b08 100644 --- a/apps/desktop/src/check-packaging-contract.ts +++ b/apps/desktop/src/check-packaging-contract.ts @@ -161,6 +161,10 @@ const promptWindowSource = readFileSync(join(appDir, "src", "prompt-window.ts"), const familiarosMemorySource = readFileSync(join(appDir, "src", "familiaros-memory.ts"), "utf8"); const familiarosMemoryStoreSource = readFileSync(join(appDir, "src", "familiaros-memory-store.ts"), "utf8"); const familiarosMemorySearchSource = readFileSync(join(appDir, "src", "familiaros-memory-search.ts"), "utf8"); +const knowledgeStoreSource = readFileSync(join(appDir, "src", "knowledge-store.ts"), "utf8"); +const knowledgeStoreCoreSource = readFileSync(join(appDir, "src", "knowledge-store-core.ts"), "utf8"); +const knowledgeStoreCoreSupportSource = readFileSync(join(appDir, "src", "knowledge-store-core-support.ts"), "utf8"); +const knowledgeStoreCoreHelpersSource = readFileSync(join(appDir, "src", "knowledge-store-core-helpers.ts"), "utf8"); const appStateSource = readFileSync(join(appDir, "src", "app-state.ts"), "utf8"); const appStatePreferencesSource = readFileSync(join(appDir, "src", "app-state-preferences.ts"), "utf8"); const appStateAnalyticsSource = readFileSync(join(appDir, "src", "app-state-analytics.ts"), "utf8"); @@ -407,6 +411,14 @@ assert.match(familiarosMemoryStoreSource, /export function pruneMemoryEntries/, assert.match(familiarosMemorySearchSource, /export function searchMemoryEntries/, "familiaros-memory search seam must export scored memory search."); assert.match(familiarosMemorySearchSource, /export function buildRelevantMemoryContextBlock/, "familiaros-memory search seam must export relevant memory context shaping."); assert.match(familiarosMemorySearchSource, /export function scoreRetentionPriority/, "familiaros-memory search seam must export retention scoring."); +assert.match(knowledgeStoreSource, /from "\.\/knowledge-store-core(?:\.js)?"/, "knowledge-store facade must continue routing through the core class seam."); +assert.match(knowledgeStoreCoreSource, /from "\.\/knowledge-store-core-support(?:\.js)?"/, "knowledge-store core must import the extracted support seam."); +assert.match(knowledgeStoreCoreSupportSource, /export function createKnowledgeFileEntry/, "knowledge-store support must own stored-file validation and shaping."); +assert.match(knowledgeStoreCoreSupportSource, /from "\.\/knowledge-store-core-helpers(?:\.js)?"/, "knowledge-store support must import the extracted helper seam."); +assert.match(knowledgeStoreCoreSupportSource, /export function searchKnowledgeFiles/, "knowledge-store support must own file search scoring."); +assert.match(knowledgeStoreCoreSupportSource, /export function buildRelevantKnowledgeContext/, "knowledge-store support must own relevant-context assembly."); +assert.match(knowledgeStoreCoreHelpersSource, /export function normalizeKnowledgeFileEntry/, "knowledge-store helpers must own index-entry normalization."); +assert.match(knowledgeStoreCoreHelpersSource, /export function scoreKnowledgeFile/, "knowledge-store helpers must own low-level search scoring."); assert.match(promptWindowRenderSource, /from "\.\/prompt-window-render-script(?:\.js)?"/, "Prompt window render helper must import the extracted inline script seam."); assert.match(promptWindowRenderScriptSource, /from "\.\/prompt-window-render-script-sections(?:\.js)?"/, "Prompt window render script seam must import the extracted script section builders."); assert.match(promptWindowRenderScriptSource, /from "\.\/prompt-window-render-script-interactions(?:\.js)?"/, "Prompt window render script seam must import the extracted interaction builder."); diff --git a/apps/desktop/src/codemap.md b/apps/desktop/src/codemap.md index 4e0eb50a..43065d6c 100644 --- a/apps/desktop/src/codemap.md +++ b/apps/desktop/src/codemap.md @@ -122,7 +122,7 @@ familiar-window preload double-click or tray/chat entry → openPromptWindow() ├── prompt-window-render.ts builds the inline prompt window HTML document/data URL │ └── prompt-window-render-script.ts owns the injected prompt window client script ├── prompt-window-preload.cjs exposes the narrow prompt window bridge -└── openapi-chat.ts, familiaros-memory.ts, familiaros-memory-store.ts, familiaros-memory-search.ts, openapi-chat-config-store.ts, openapi-chat-request-helpers.ts, openapi-chat-provider.ts, and knowledge-store.ts handle prompt submission, long-term memory recall, credential persistence, request shaping, provider routing, history, and attachment storage +└── openapi-chat.ts, familiaros-memory.ts, familiaros-memory-store.ts, familiaros-memory-search.ts, openapi-chat-config-store.ts, openapi-chat-request-helpers.ts, openapi-chat-provider.ts, knowledge-store.ts, and knowledge-store-core.ts handle prompt submission, long-term memory recall, knowledge-file storage/search, credential persistence, request shaping, provider routing, history, and attachment storage ``` **Plugin Flow**: @@ -219,6 +219,10 @@ plugin-service-local-support.ts → plugin-local-loader.ts validates selected fo - `familiaros-memory.ts`: Public long-term memory API, CRUD orchestration, context shaping, and prompt-memory capture for chat, IPC, and knowledge-store flows - `familiaros-memory-store.ts`: Extracted memory cache/persistence, JSON and markdown mirror writes, normalization, retention pruning, and mutation helpers - `familiaros-memory-search.ts`: Extracted pure memory query normalization, scoring, recall-intent fallback, chat-history relevance, and context-block shaping for FamiliarOS memory +- `knowledge-store.ts`: Public knowledge-store facade that combines file search/storage with FamiliarOS memory search results +- `knowledge-store-core.ts`: Extracted disk-backed knowledge file store class for CRUD, index caching, and path lookup +- `knowledge-store-core-support.ts`: Extracted knowledge-store public support seam for file validation, search/context assembly, normalization, and atomic writes +- `knowledge-store-core-helpers.ts`: Low-level knowledge-store parsing, MIME/text inference, query scoring, and tokenization helpers used by the public support seam - `openapi-chat-tool-loop.ts`: Extracted chat-completions tool-call loop, history seeding, tool-result threading, and tool-aware instruction helpers for prompt-window OpenAPI chat - `openapi-chat-config-store.ts`: Extracted credential persistence, storage-mode selection, current/legacy config path handling, and stored-credential decode helpers for prompt-window OpenAPI chat - `openapi-chat-request-helpers.ts`: Extracted pure request-body and chat-completions history shaping for prompt-window OpenAPI chat diff --git a/apps/desktop/src/knowledge-store-core-helpers.ts b/apps/desktop/src/knowledge-store-core-helpers.ts new file mode 100644 index 00000000..b760ca19 --- /dev/null +++ b/apps/desktop/src/knowledge-store-core-helpers.ts @@ -0,0 +1,235 @@ +import { Buffer } from "node:buffer"; +import { TextDecoder } from "node:util"; + +import type { KnowledgeFileEntry } from "./knowledge-store-core-support.js"; + +export const maxKnowledgeFileSizeBytes = 5 * 1024 * 1024; + +const maxExtractedTextChars = 2_000; +const maxFileNameLength = 128; + +export function normalizeKnowledgeFileEntry( + value: unknown, +): KnowledgeFileEntry | undefined { + if ( + !isRecord(value) || + typeof value.id !== "string" || + !value.id.startsWith("kfile-") + ) { + return undefined; + } + const name = + sanitizeFileName(value.name) ?? + sanitizeFileName(value.originalName) ?? + "unnamed"; + const originalName = sanitizeFileName(value.originalName) ?? name; + const mimeType = + typeof value.mimeType === "string" && value.mimeType.includes("/") + ? value.mimeType + : inferMimeType(originalName); + const size = + typeof value.size === "number" && + Number.isFinite(value.size) && + value.size > 0 + ? Math.floor(value.size) + : 0; + const isText = value.isText === true; + const extractedText = + isText && typeof value.extractedText === "string" + ? trimExtractedText(value.extractedText) + : undefined; + const addedAt = + typeof value.addedAt === "number" && + Number.isFinite(value.addedAt) && + value.addedAt > 0 + ? Math.floor(value.addedAt) + : Date.now(); + return { + id: value.id, + name, + originalName, + mimeType, + size, + isText, + extractedText, + extractedTextLength: extractedText?.length ?? 0, + addedAt, + }; +} + +export function scoreKnowledgeFile( + file: KnowledgeFileEntry, + queryTokens: ReadonlySet, + normalizedQuery: string, +): number { + const nameTokens = tokenize(file.name); + let overlapCount = 0; + for (const token of queryTokens) { + if (nameTokens.has(token)) { + overlapCount += 1; + } + } + const textTokens = file.extractedText + ? tokenize(file.extractedText) + : new Set(); + for (const token of queryTokens) { + if (textTokens.has(token)) { + overlapCount += 0.5; + } + } + + const normalizedName = normalizeKnowledgeComparisonValue(file.name); + const normalizedText = normalizeKnowledgeComparisonValue( + file.extractedText ?? "", + ); + const phraseBonus = + normalizedName.includes(normalizedQuery) || + normalizedText.includes(normalizedQuery) + ? 2 + : 0; + if (overlapCount === 0 && phraseBonus === 0) { + return 0; + } + const recencyBonus = + Math.max(0, 1 - (Date.now() - file.addedAt) / (30 * 24 * 60 * 60 * 1000)) * + 0.3; + const sizePenalty = + Math.max(0, file.size / maxKnowledgeFileSizeBytes) * 0.2; + return overlapCount + phraseBonus + recencyBonus - sizePenalty; +} + +export function extractTextContent(data: Buffer): string { + const raw = data.toString("utf8"); + const normalized = raw.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + return trimExtractedText(normalized); +} + +export function sanitizeFileName(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const name = value.replace(/\\/g, "/").split("/").pop() ?? ""; + const sanitized = name + .replace(/[\x00-\x1f\x7f<>|:*"?\\]/g, "_") + .trim(); + if (!sanitized || sanitized.length > maxFileNameLength) return undefined; + return sanitized; +} + +export function trimFileName(name: string): string { + if (name.length <= maxFileNameLength) return name; + const extension = extname(name); + const base = name.slice(0, name.length - extension.length); + const maxBaseLength = maxFileNameLength - extension.length - 1; + return `${base.slice(0, Math.max(0, maxBaseLength))}…${extension}`; +} + +export function inferMimeType(fileName: string): string { + const ext = extname(fileName).toLowerCase(); + switch (ext) { + case ".txt": + return "text/plain"; + case ".md": + return "text/markdown"; + case ".json": + return "application/json"; + case ".js": + return "text/javascript"; + case ".ts": + return "text/typescript"; + case ".jsx": + return "text/jsx"; + case ".tsx": + return "text/tsx"; + case ".css": + return "text/css"; + case ".html": + return "text/html"; + case ".xml": + return "text/xml"; + case ".yaml": + case ".yml": + return "text/yaml"; + case ".csv": + return "text/csv"; + case ".pdf": + return "application/pdf"; + case ".png": + return "image/png"; + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".gif": + return "image/gif"; + case ".webp": + return "image/webp"; + case ".svg": + return "image/svg+xml"; + case ".mp3": + return "audio/mpeg"; + case ".mp4": + return "video/mp4"; + default: + return "application/octet-stream"; + } +} + +export function normalizeKnowledgeQuery(value: string): string { + return value.replace(/\s+/g, " ").trim().toLowerCase(); +} + +export function normalizeKnowledgeComparisonValue(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +export function tokenize(value: string): Set { + return new Set( + normalizeKnowledgeComparisonValue(value) + .split(" ") + .map((token) => token.trim()) + .filter((token) => token.length >= 2), + ); +} + +export function isValidUtf8Buffer(data: Buffer): boolean { + const bufferConstructor = Buffer as unknown as { + isUtf8?(data: Buffer): boolean; + }; + if (typeof bufferConstructor.isUtf8 === "function") { + return bufferConstructor.isUtf8(data); + } + try { + new TextDecoder("utf-8", { fatal: true }).decode(data); + return true; + } catch { + return false; + } +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function sortKnowledgeFiles( + files: readonly KnowledgeFileEntry[], +): readonly KnowledgeFileEntry[] { + return [...files].sort( + (left, right) => + right.addedAt - left.addedAt || left.name.localeCompare(right.name), + ); +} + +function trimExtractedText(text: string): string { + const trimmed = text.replace(/[ \t]+/g, " ").trim(); + if (trimmed.length <= maxExtractedTextChars) { + return trimmed; + } + return `${trimmed.slice(0, maxExtractedTextChars)}\n…`; +} + +function extname(name: string): string { + const dotIndex = name.lastIndexOf("."); + return dotIndex <= 0 ? "" : name.slice(dotIndex); +} diff --git a/apps/desktop/src/knowledge-store-core-support.ts b/apps/desktop/src/knowledge-store-core-support.ts new file mode 100644 index 00000000..544fb08e --- /dev/null +++ b/apps/desktop/src/knowledge-store-core-support.ts @@ -0,0 +1,228 @@ +import { Buffer } from "node:buffer"; +import { mkdir, rename, writeFile } from "node:fs/promises"; +import { dirname, extname } from "node:path"; + +import { + extractTextContent, + inferMimeType, + isRecord, + isValidUtf8Buffer, + maxKnowledgeFileSizeBytes, + normalizeKnowledgeComparisonValue, + normalizeKnowledgeFileEntry as parseKnowledgeFileEntry, + normalizeKnowledgeQuery, + sanitizeFileName, + scoreKnowledgeFile, + sortKnowledgeFiles, + tokenize, + trimFileName, +} from "./knowledge-store-core-helpers.js"; + +export interface KnowledgeFileEntry { + readonly id: string; + readonly name: string; + readonly originalName: string; + readonly mimeType: string; + readonly size: number; + readonly isText: boolean; + readonly extractedText: string | undefined; + readonly extractedTextLength: number; + readonly addedAt: number; +} + +export interface KnowledgeFileSearchHit { + readonly file: KnowledgeFileEntry; + readonly score: number; +} + +export interface StoreKnowledgeFileInput { + readonly name: string; + readonly data: Buffer; + readonly mimeType?: string; +} + +export interface KnowledgeStoreIndexV1 { + readonly version: 1; + readonly files: readonly KnowledgeFileEntry[]; +} + +export const knowledgeStoreVersion = 1; + +const maxKnowledgeFiles = 100; +const maxRelevantKnowledgeItems = 4; +const maxRelevantKnowledgeChars = 2_000; + +export function clampKnowledgeLimit(value: number, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + return Math.min(Math.max(Math.round(value), 1), 100); +} + +export function cloneKnowledgeFileEntry( + entry: KnowledgeFileEntry, +): KnowledgeFileEntry { + return { ...entry }; +} + +export function createKnowledgeFileEntry( + id: string, + input: StoreKnowledgeFileInput, + addedAt = Date.now(), +): { readonly entry: KnowledgeFileEntry; readonly extension: string } { + const originalName = sanitizeFileName(input.name); + if (!originalName) { + throw new Error("File name is required."); + } + if (!Buffer.isBuffer(input.data)) { + throw new Error("File data must be a Buffer."); + } + if (input.data.length === 0) { + throw new Error("File cannot be empty."); + } + if (input.data.length > maxKnowledgeFileSizeBytes) { + throw new Error( + `File is too large. Maximum size is ${formatBytes(maxKnowledgeFileSizeBytes)}.`, + ); + } + + const displayName = trimFileName(originalName); + const extension = extname(originalName).toLowerCase(); + const mimeType = + input.mimeType && input.mimeType.includes("/") + ? input.mimeType.trim() + : inferMimeType(originalName); + const isText = isValidUtf8Buffer(input.data) && !input.data.includes(0); + const extractedText = isText ? extractTextContent(input.data) : undefined; + + return { + extension, + entry: { + id, + name: displayName, + originalName, + mimeType, + size: input.data.length, + isText, + extractedText, + extractedTextLength: extractedText?.length ?? 0, + addedAt, + }, + }; +} + +export function searchKnowledgeFiles( + files: readonly KnowledgeFileEntry[], + query: string, + limit = 10, +): readonly KnowledgeFileSearchHit[] { + const trimmedQuery = normalizeKnowledgeQuery(query); + const nextLimit = clampKnowledgeLimit(limit, 10); + if (!trimmedQuery) { + return sortKnowledgeFiles(files) + .slice(0, nextLimit) + .map((file) => ({ file: cloneKnowledgeFileEntry(file), score: 0 })); + } + + const queryTokens = tokenize(trimmedQuery); + const normalizedQuery = normalizeKnowledgeComparisonValue(trimmedQuery); + return files + .map((file) => ({ + file, + score: scoreKnowledgeFile(file, queryTokens, normalizedQuery), + })) + .filter((hit) => hit.score > 0) + .sort( + (left, right) => + right.score - left.score || + right.file.addedAt - left.file.addedAt || + left.file.name.localeCompare(right.file.name), + ) + .slice(0, nextLimit) + .map((hit) => ({ + file: cloneKnowledgeFileEntry(hit.file), + score: Math.round(hit.score * 100) / 100, + })); +} + +export function buildRelevantKnowledgeContext( + files: readonly KnowledgeFileEntry[], + prompt: string, +): string | undefined { + const trimmedQuery = normalizeKnowledgeQuery(prompt); + if (!trimmedQuery) { + return undefined; + } + + const hits = searchKnowledgeFiles( + files, + trimmedQuery, + maxRelevantKnowledgeItems, + ).filter((hit) => hit.score > 0); + if (hits.length === 0) { + return undefined; + } + + const lines: string[] = []; + let usedChars = 0; + for (const hit of hits) { + const file = hit.file; + const header = `- [file: ${file.name}] (${formatBytes(file.size)}, ${file.mimeType})`; + const body = + file.isText && file.extractedText + ? file.extractedText + : "[Non-text file: content not shown.]"; + const entry = `${header}\n${body}`; + if (usedChars > 0 && usedChars + entry.length + 2 > maxRelevantKnowledgeChars) { + break; + } + lines.push(entry); + usedChars += entry.length + 2; + } + + if (lines.length === 0) { + return undefined; + } + + return [ + "Relevant stored files:", + ...lines, + "Use the above files only when they help answer the current request.", + ].join("\n\n"); +} + +export function normalizeKnowledgeStore(value: unknown): KnowledgeStoreIndexV1 { + const record = isRecord(value) ? value : {}; + const files = Array.isArray(record.files) + ? record.files + .map((file) => parseKnowledgeFileEntry(file)) + .filter((file): file is KnowledgeFileEntry => Boolean(file)) + : []; + return { + version: 1, + files: pruneKnowledgeFiles(files), + }; +} + +export function pruneKnowledgeFiles( + files: readonly KnowledgeFileEntry[], +): readonly KnowledgeFileEntry[] { + if (files.length <= maxKnowledgeFiles) { + return [...files]; + } + return sortKnowledgeFiles(files).slice(0, maxKnowledgeFiles); +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export async function atomicWrite( + targetPath: string, + data: Buffer, +): Promise { + await mkdir(dirname(targetPath), { recursive: true }); + const tempPath = `${targetPath}.${process.pid}.tmp`; + await writeFile(tempPath, data); + await rename(tempPath, targetPath); +} diff --git a/apps/desktop/src/knowledge-store-core.ts b/apps/desktop/src/knowledge-store-core.ts index de20010b..0687af9d 100644 --- a/apps/desktop/src/knowledge-store-core.ts +++ b/apps/desktop/src/knowledge-store-core.ts @@ -1,48 +1,34 @@ import { randomUUID } from "node:crypto"; import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; -import { mkdir, rename, unlink, writeFile } from "node:fs/promises"; -import { Buffer } from "node:buffer"; -import { TextDecoder } from "node:util"; +import { unlink } from "node:fs/promises"; import { dirname, extname, join } from "node:path"; -export interface KnowledgeFileEntry { - readonly id: string; - readonly name: string; - readonly originalName: string; - readonly mimeType: string; - readonly size: number; - readonly isText: boolean; - readonly extractedText: string | undefined; - readonly extractedTextLength: number; - readonly addedAt: number; -} +import { + atomicWrite, + buildRelevantKnowledgeContext, + clampKnowledgeLimit, + cloneKnowledgeFileEntry, + createKnowledgeFileEntry, + knowledgeStoreVersion, + normalizeKnowledgeStore, + pruneKnowledgeFiles, + searchKnowledgeFiles, + type KnowledgeFileEntry, + type KnowledgeFileSearchHit, + type KnowledgeStoreIndexV1, + type StoreKnowledgeFileInput, +} from "./knowledge-store-core-support.js"; -export interface KnowledgeFileSearchHit { - readonly file: KnowledgeFileEntry; - readonly score: number; -} +export { formatBytes } from "./knowledge-store-core-support.js"; +export type { + KnowledgeFileEntry, + KnowledgeFileSearchHit, + StoreKnowledgeFileInput, +} from "./knowledge-store-core-support.js"; -export interface StoreKnowledgeFileInput { - readonly name: string; - readonly data: Buffer; - readonly mimeType?: string; -} - -const knowledgeStoreVersion = 1; -const maxKnowledgeFiles = 100; -const maxKnowledgeFileSizeBytes = 5 * 1024 * 1024; -const maxExtractedTextChars = 2_000; -const maxRelevantKnowledgeItems = 4; -const maxRelevantKnowledgeChars = 2_000; -const maxFileNameLength = 128; const knowledgeFilesDirName = "files"; const knowledgeStoreIndexFileName = "knowledge-store.json"; -interface KnowledgeStoreIndexV1 { - readonly version: 1; - readonly files: readonly KnowledgeFileEntry[]; -} - export class KnowledgeStore { private cachedStore: KnowledgeStoreIndexV1 | null = null; @@ -51,68 +37,23 @@ export class KnowledgeStore { listFiles(limit = 50): readonly KnowledgeFileEntry[] { const nextLimit = clampKnowledgeLimit(limit, 50); return [...this.getStore().files] - .sort((left, right) => right.addedAt - left.addedAt || left.name.localeCompare(right.name)) + .sort( + (left, right) => + right.addedAt - left.addedAt || left.name.localeCompare(right.name), + ) .slice(0, nextLimit) .map(cloneKnowledgeFileEntry); } searchFiles(query: string, limit = 10): readonly KnowledgeFileSearchHit[] { - const trimmedQuery = normalizeKnowledgeQuery(query); - const nextLimit = clampKnowledgeLimit(limit, 10); - if (!trimmedQuery) { - return this.listFiles(nextLimit).map((file) => ({ file, score: 0 })); - } - - const queryTokens = tokenize(trimmedQuery); - const normalizedQuery = normalizeKnowledgeComparisonValue(trimmedQuery); - return this.getStore().files - .map((file) => ({ file, score: scoreKnowledgeFile(file, queryTokens, normalizedQuery) })) - .filter((hit) => hit.score > 0) - .sort((left, right) => right.score - left.score || right.file.addedAt - left.file.addedAt || left.file.name.localeCompare(right.file.name)) - .slice(0, nextLimit) - .map((hit) => ({ file: cloneKnowledgeFileEntry(hit.file), score: Math.round(hit.score * 100) / 100 })); + return searchKnowledgeFiles(this.getStore().files, query, limit); } async storeFile(input: StoreKnowledgeFileInput): Promise { - const originalName = sanitizeFileName(input.name); - if (!originalName) { - throw new Error("File name is required."); - } - if (!Buffer.isBuffer(input.data)) { - throw new Error("File data must be a Buffer."); - } - if (input.data.length === 0) { - throw new Error("File cannot be empty."); - } - if (input.data.length > maxKnowledgeFileSizeBytes) { - throw new Error(`File is too large. Maximum size is ${formatBytes(maxKnowledgeFileSizeBytes)}.`); - } - const id = `kfile-${randomUUID()}`; - const displayName = trimFileName(originalName); - const extension = extname(originalName).toLowerCase(); - const mimeType = input.mimeType && input.mimeType.includes("/") - ? input.mimeType.trim() - : inferMimeType(originalName); - - const isText = isValidUtf8Buffer(input.data) && !input.data.includes(0); - const extractedText = isText ? extractTextContent(input.data) : undefined; - - const entry: KnowledgeFileEntry = { - id, - name: displayName, - originalName, - mimeType, - size: input.data.length, - isText, - extractedText, - extractedTextLength: extractedText?.length ?? 0, - addedAt: Date.now(), - }; - + const { entry, extension } = createKnowledgeFileEntry(id, input); const store = this.getStore(); const filesDir = this.getFilesDir(); - await mkdir(filesDir, { recursive: true }); const filePath = join(filesDir, `${id}${extension}`); await atomicWrite(filePath, input.data); @@ -154,38 +95,7 @@ export class KnowledgeStore { } buildRelevantContext(prompt: string): string | undefined { - const trimmedQuery = normalizeKnowledgeQuery(prompt); - if (!trimmedQuery) { - return undefined; - } - const hits = this.searchFiles(trimmedQuery, maxRelevantKnowledgeItems).filter((hit) => hit.score > 0); - if (hits.length === 0) { - return undefined; - } - - const lines: string[] = []; - let usedChars = 0; - for (const hit of hits) { - const file = hit.file; - const header = `- [file: ${file.name}] (${formatBytes(file.size)}, ${file.mimeType})`; - const body = file.isText && file.extractedText ? file.extractedText : "[Non-text file: content not shown.]"; - const entry = `${header}\n${body}`; - if (usedChars > 0 && usedChars + entry.length + 2 > maxRelevantKnowledgeChars) { - break; - } - lines.push(entry); - usedChars += entry.length + 2; - } - - if (lines.length === 0) { - return undefined; - } - - return [ - "Relevant stored files:", - ...lines, - "Use the above files only when they help answer the current request.", - ].join("\n\n"); + return buildRelevantKnowledgeContext(this.getStore().files, prompt); } getFilePath(id: string): string | null { @@ -205,14 +115,14 @@ export class KnowledgeStore { const raw = JSON.parse(readFileSync(this.getIndexPath(), "utf8")) as unknown; this.cachedStore = normalizeKnowledgeStore(raw); } catch { - this.cachedStore = { version: 1, files: [] }; + this.cachedStore = { version: knowledgeStoreVersion, files: [] }; } return this.cachedStore; } private commitStore(store: KnowledgeStoreIndexV1): void { this.cachedStore = { - version: 1, + version: knowledgeStoreVersion, files: store.files.map(cloneKnowledgeFileEntry), }; const path = this.getIndexPath(); @@ -231,188 +141,3 @@ export class KnowledgeStore { return join(this.baseDir, knowledgeStoreIndexFileName); } } - -export function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; -} - -function normalizeKnowledgeStore(value: unknown): KnowledgeStoreIndexV1 { - const record = isRecord(value) ? value : {}; - const files = Array.isArray(record.files) - ? record.files.map((file) => normalizeKnowledgeFileEntry(file)).filter((file): file is KnowledgeFileEntry => Boolean(file)) - : []; - return { - version: 1, - files: pruneKnowledgeFiles(files), - }; -} - -function normalizeKnowledgeFileEntry(value: unknown): KnowledgeFileEntry | undefined { - if (!isRecord(value) || typeof value.id !== "string" || !value.id.startsWith("kfile-")) { - return undefined; - } - const name = sanitizeFileName(value.name) ?? sanitizeFileName(value.originalName) ?? "unnamed"; - const originalName = sanitizeFileName(value.originalName) ?? name; - const mimeType = typeof value.mimeType === "string" && value.mimeType.includes("/") ? value.mimeType : inferMimeType(originalName); - const size = typeof value.size === "number" && Number.isFinite(value.size) && value.size > 0 ? Math.floor(value.size) : 0; - const isText = value.isText === true; - const extractedText = isText && typeof value.extractedText === "string" ? trimExtractedText(value.extractedText) : undefined; - const addedAt = typeof value.addedAt === "number" && Number.isFinite(value.addedAt) && value.addedAt > 0 ? Math.floor(value.addedAt) : Date.now(); - return { - id: value.id, - name, - originalName, - mimeType, - size, - isText, - extractedText, - extractedTextLength: extractedText?.length ?? 0, - addedAt, - }; -} - -function pruneKnowledgeFiles(files: readonly KnowledgeFileEntry[]): readonly KnowledgeFileEntry[] { - if (files.length <= maxKnowledgeFiles) { - return [...files]; - } - return [...files] - .sort((left, right) => right.addedAt - left.addedAt) - .slice(0, maxKnowledgeFiles); -} - -function scoreKnowledgeFile(file: KnowledgeFileEntry, queryTokens: ReadonlySet, normalizedQuery: string): number { - const nameTokens = tokenize(file.name); - let overlapCount = 0; - for (const token of queryTokens) { - if (nameTokens.has(token)) { - overlapCount += 1; - } - } - const textTokens = file.extractedText ? tokenize(file.extractedText) : new Set(); - for (const token of queryTokens) { - if (textTokens.has(token)) { - overlapCount += 0.5; - } - } - - const normalizedName = normalizeKnowledgeComparisonValue(file.name); - const normalizedText = normalizeKnowledgeComparisonValue(file.extractedText ?? ""); - const phraseBonus = normalizedName.includes(normalizedQuery) || normalizedText.includes(normalizedQuery) ? 2 : 0; - if (overlapCount === 0 && phraseBonus === 0) { - return 0; - } - const recencyBonus = Math.max(0, 1 - (Date.now() - file.addedAt) / (30 * 24 * 60 * 60 * 1000)) * 0.3; - const sizePenalty = Math.max(0, file.size / maxKnowledgeFileSizeBytes) * 0.2; - return overlapCount + phraseBonus + recencyBonus - sizePenalty; -} - -function extractTextContent(data: Buffer): string { - const raw = data.toString("utf8"); - const normalized = raw.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); - return trimExtractedText(normalized); -} - -function trimExtractedText(text: string): string { - const trimmed = text.replace(/[ \t]+/g, " ").trim(); - if (trimmed.length <= maxExtractedTextChars) { - return trimmed; - } - return `${trimmed.slice(0, maxExtractedTextChars)}\n…`; -} - -function sanitizeFileName(value: unknown): string | undefined { - if (typeof value !== "string") return undefined; - const name = value.replace(/\\/g, "/").split("/").pop() ?? ""; - const sanitized = name.replace(/[\x00-\x1f\x7f<>|:*"?\\]/g, "_").trim(); - if (!sanitized || sanitized.length > maxFileNameLength) return undefined; - return sanitized; -} - -function trimFileName(name: string): string { - if (name.length <= maxFileNameLength) return name; - const extension = extname(name); - const base = name.slice(0, name.length - extension.length); - const maxBaseLength = maxFileNameLength - extension.length - 1; - return `${base.slice(0, Math.max(0, maxBaseLength))}…${extension}`; -} - -function inferMimeType(fileName: string): string { - const ext = extname(fileName).toLowerCase(); - switch (ext) { - case ".txt": return "text/plain"; - case ".md": return "text/markdown"; - case ".json": return "application/json"; - case ".js": return "text/javascript"; - case ".ts": return "text/typescript"; - case ".jsx": return "text/jsx"; - case ".tsx": return "text/tsx"; - case ".css": return "text/css"; - case ".html": return "text/html"; - case ".xml": return "text/xml"; - case ".yaml": - case ".yml": return "text/yaml"; - case ".csv": return "text/csv"; - case ".pdf": return "application/pdf"; - case ".png": return "image/png"; - case ".jpg": - case ".jpeg": return "image/jpeg"; - case ".gif": return "image/gif"; - case ".webp": return "image/webp"; - case ".svg": return "image/svg+xml"; - case ".mp3": return "audio/mpeg"; - case ".mp4": return "video/mp4"; - default: return "application/octet-stream"; - } -} - -function cloneKnowledgeFileEntry(entry: KnowledgeFileEntry): KnowledgeFileEntry { - return { ...entry }; -} - -function clampKnowledgeLimit(value: number, fallback: number): number { - if (!Number.isFinite(value)) return fallback; - return Math.min(Math.max(Math.round(value), 1), 100); -} - -function normalizeKnowledgeQuery(value: string): string { - return value.replace(/\s+/g, " ").trim().toLowerCase(); -} - -function normalizeKnowledgeComparisonValue(value: string): string { - return value.toLowerCase().replace(/[^a-z0-9\s]+/g, " ").replace(/\s+/g, " ").trim(); -} - -function tokenize(value: string): Set { - return new Set( - normalizeKnowledgeComparisonValue(value) - .split(" ") - .map((token) => token.trim()) - .filter((token) => token.length >= 2), - ); -} - -function isValidUtf8Buffer(data: Buffer): boolean { - const bufferConstructor = Buffer as unknown as { isUtf8?(data: Buffer): boolean }; - if (typeof bufferConstructor.isUtf8 === "function") { - return bufferConstructor.isUtf8(data); - } - try { - new TextDecoder("utf-8", { fatal: true }).decode(data); - return true; - } catch { - return false; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -async function atomicWrite(targetPath: string, data: Buffer): Promise { - await mkdir(dirname(targetPath), { recursive: true }); - const tempPath = `${targetPath}.${process.pid}.tmp`; - await writeFile(tempPath, data); - await rename(tempPath, targetPath); -} diff --git a/apps/desktop/tests/knowledge-store-core-seam.test.ts b/apps/desktop/tests/knowledge-store-core-seam.test.ts new file mode 100644 index 00000000..b307f173 --- /dev/null +++ b/apps/desktop/tests/knowledge-store-core-seam.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const desktopRoot = + process.env.FAMILIAROS_DESKTOP_ROOT ?? + resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const knowledgeStoreCoreSource = readFileSync( + resolve(desktopRoot, "src/knowledge-store-core.ts"), + "utf8", +); +const knowledgeStoreCoreSupportSource = readFileSync( + resolve(desktopRoot, "src/knowledge-store-core-support.ts"), + "utf8", +); +const knowledgeStoreCoreHelpersSource = readFileSync( + resolve(desktopRoot, "src/knowledge-store-core-helpers.ts"), + "utf8", +); + +assert.match( + knowledgeStoreCoreSource, + /from "\.\/knowledge-store-core-support(?:\.js)?"/, + "knowledge-store core must import the extracted support seam.", +); +assert.match( + knowledgeStoreCoreSource, + /export class KnowledgeStore/, + "knowledge-store core must keep exporting the public KnowledgeStore class.", +); +assert.match( + knowledgeStoreCoreSupportSource, + /export function createKnowledgeFileEntry/, + "knowledge-store support must own stored-file validation and shaping.", +); +assert.match( + knowledgeStoreCoreSupportSource, + /from "\.\/knowledge-store-core-helpers(?:\.js)?"/, + "knowledge-store support must import the extracted helper seam.", +); +assert.match( + knowledgeStoreCoreSupportSource, + /export function searchKnowledgeFiles/, + "knowledge-store support must own file search scoring.", +); +assert.match( + knowledgeStoreCoreSupportSource, + /export function buildRelevantKnowledgeContext/, + "knowledge-store support must own relevant-context assembly.", +); +assert.match( + knowledgeStoreCoreHelpersSource, + /export function normalizeKnowledgeFileEntry/, + "knowledge-store helpers must own index-entry normalization.", +); +assert.match( + knowledgeStoreCoreHelpersSource, + /export function scoreKnowledgeFile/, + "knowledge-store helpers must own low-level search scoring.", +); + +console.error("Knowledge store core seam validation passed.");