Extract knowledge store core helpers
This commit is contained in:
OpenPets Dev
parent
9e75151e56
commit
2b1da0c5ea
7 changed files with 575 additions and 307 deletions
|
|
@ -61,6 +61,7 @@ const behaviorTests = [
|
|||
".test-dist/tests/claude-memory.test.js",
|
||||
".test-dist/tests/prompt-memory-extraction.test.js",
|
||||
".test-dist/tests/familiaros-memory-search.test.js",
|
||||
".test-dist/tests/knowledge-store-core-seam.test.js",
|
||||
".test-dist/tests/knowledge-store.test.js",
|
||||
".test-dist/tests/control-center-service-barrels.test.js",
|
||||
".test-dist/tests/mcp-toolkit-catalog.test.js",
|
||||
|
|
|
|||
|
|
@ -161,6 +161,10 @@ const promptWindowSource = readFileSync(join(appDir, "src", "prompt-window.ts"),
|
|||
const familiarosMemorySource = readFileSync(join(appDir, "src", "familiaros-memory.ts"), "utf8");
|
||||
const familiarosMemoryStoreSource = readFileSync(join(appDir, "src", "familiaros-memory-store.ts"), "utf8");
|
||||
const familiarosMemorySearchSource = readFileSync(join(appDir, "src", "familiaros-memory-search.ts"), "utf8");
|
||||
const knowledgeStoreSource = readFileSync(join(appDir, "src", "knowledge-store.ts"), "utf8");
|
||||
const knowledgeStoreCoreSource = readFileSync(join(appDir, "src", "knowledge-store-core.ts"), "utf8");
|
||||
const knowledgeStoreCoreSupportSource = readFileSync(join(appDir, "src", "knowledge-store-core-support.ts"), "utf8");
|
||||
const knowledgeStoreCoreHelpersSource = readFileSync(join(appDir, "src", "knowledge-store-core-helpers.ts"), "utf8");
|
||||
const appStateSource = readFileSync(join(appDir, "src", "app-state.ts"), "utf8");
|
||||
const appStatePreferencesSource = readFileSync(join(appDir, "src", "app-state-preferences.ts"), "utf8");
|
||||
const appStateAnalyticsSource = readFileSync(join(appDir, "src", "app-state-analytics.ts"), "utf8");
|
||||
|
|
@ -407,6 +411,14 @@ assert.match(familiarosMemoryStoreSource, /export function pruneMemoryEntries/,
|
|||
assert.match(familiarosMemorySearchSource, /export function searchMemoryEntries/, "familiaros-memory search seam must export scored memory search.");
|
||||
assert.match(familiarosMemorySearchSource, /export function buildRelevantMemoryContextBlock/, "familiaros-memory search seam must export relevant memory context shaping.");
|
||||
assert.match(familiarosMemorySearchSource, /export function scoreRetentionPriority/, "familiaros-memory search seam must export retention scoring.");
|
||||
assert.match(knowledgeStoreSource, /from "\.\/knowledge-store-core(?:\.js)?"/, "knowledge-store facade must continue routing through the core class seam.");
|
||||
assert.match(knowledgeStoreCoreSource, /from "\.\/knowledge-store-core-support(?:\.js)?"/, "knowledge-store core must import the extracted support seam.");
|
||||
assert.match(knowledgeStoreCoreSupportSource, /export function createKnowledgeFileEntry/, "knowledge-store support must own stored-file validation and shaping.");
|
||||
assert.match(knowledgeStoreCoreSupportSource, /from "\.\/knowledge-store-core-helpers(?:\.js)?"/, "knowledge-store support must import the extracted helper seam.");
|
||||
assert.match(knowledgeStoreCoreSupportSource, /export function searchKnowledgeFiles/, "knowledge-store support must own file search scoring.");
|
||||
assert.match(knowledgeStoreCoreSupportSource, /export function buildRelevantKnowledgeContext/, "knowledge-store support must own relevant-context assembly.");
|
||||
assert.match(knowledgeStoreCoreHelpersSource, /export function normalizeKnowledgeFileEntry/, "knowledge-store helpers must own index-entry normalization.");
|
||||
assert.match(knowledgeStoreCoreHelpersSource, /export function scoreKnowledgeFile/, "knowledge-store helpers must own low-level search scoring.");
|
||||
assert.match(promptWindowRenderSource, /from "\.\/prompt-window-render-script(?:\.js)?"/, "Prompt window render helper must import the extracted inline script seam.");
|
||||
assert.match(promptWindowRenderScriptSource, /from "\.\/prompt-window-render-script-sections(?:\.js)?"/, "Prompt window render script seam must import the extracted script section builders.");
|
||||
assert.match(promptWindowRenderScriptSource, /from "\.\/prompt-window-render-script-interactions(?:\.js)?"/, "Prompt window render script seam must import the extracted interaction builder.");
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ familiar-window preload double-click or tray/chat entry → openPromptWindow()
|
|||
├── prompt-window-render.ts builds the inline prompt window HTML document/data URL
|
||||
│ └── prompt-window-render-script.ts owns the injected prompt window client script
|
||||
├── prompt-window-preload.cjs exposes the narrow prompt window bridge
|
||||
└── openapi-chat.ts, familiaros-memory.ts, familiaros-memory-store.ts, familiaros-memory-search.ts, openapi-chat-config-store.ts, openapi-chat-request-helpers.ts, openapi-chat-provider.ts, and knowledge-store.ts handle prompt submission, long-term memory recall, credential persistence, request shaping, provider routing, history, and attachment storage
|
||||
└── openapi-chat.ts, familiaros-memory.ts, familiaros-memory-store.ts, familiaros-memory-search.ts, openapi-chat-config-store.ts, openapi-chat-request-helpers.ts, openapi-chat-provider.ts, knowledge-store.ts, and knowledge-store-core.ts handle prompt submission, long-term memory recall, knowledge-file storage/search, credential persistence, request shaping, provider routing, history, and attachment storage
|
||||
```
|
||||
|
||||
**Plugin Flow**:
|
||||
|
|
@ -219,6 +219,10 @@ plugin-service-local-support.ts → plugin-local-loader.ts validates selected fo
|
|||
- `familiaros-memory.ts`: Public long-term memory API, CRUD orchestration, context shaping, and prompt-memory capture for chat, IPC, and knowledge-store flows
|
||||
- `familiaros-memory-store.ts`: Extracted memory cache/persistence, JSON and markdown mirror writes, normalization, retention pruning, and mutation helpers
|
||||
- `familiaros-memory-search.ts`: Extracted pure memory query normalization, scoring, recall-intent fallback, chat-history relevance, and context-block shaping for FamiliarOS memory
|
||||
- `knowledge-store.ts`: Public knowledge-store facade that combines file search/storage with FamiliarOS memory search results
|
||||
- `knowledge-store-core.ts`: Extracted disk-backed knowledge file store class for CRUD, index caching, and path lookup
|
||||
- `knowledge-store-core-support.ts`: Extracted knowledge-store public support seam for file validation, search/context assembly, normalization, and atomic writes
|
||||
- `knowledge-store-core-helpers.ts`: Low-level knowledge-store parsing, MIME/text inference, query scoring, and tokenization helpers used by the public support seam
|
||||
- `openapi-chat-tool-loop.ts`: Extracted chat-completions tool-call loop, history seeding, tool-result threading, and tool-aware instruction helpers for prompt-window OpenAPI chat
|
||||
- `openapi-chat-config-store.ts`: Extracted credential persistence, storage-mode selection, current/legacy config path handling, and stored-credential decode helpers for prompt-window OpenAPI chat
|
||||
- `openapi-chat-request-helpers.ts`: Extracted pure request-body and chat-completions history shaping for prompt-window OpenAPI chat
|
||||
|
|
|
|||
235
apps/desktop/src/knowledge-store-core-helpers.ts
Normal file
235
apps/desktop/src/knowledge-store-core-helpers.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import { Buffer } from "node:buffer";
|
||||
import { TextDecoder } from "node:util";
|
||||
|
||||
import type { KnowledgeFileEntry } from "./knowledge-store-core-support.js";
|
||||
|
||||
export const maxKnowledgeFileSizeBytes = 5 * 1024 * 1024;
|
||||
|
||||
const maxExtractedTextChars = 2_000;
|
||||
const maxFileNameLength = 128;
|
||||
|
||||
export function normalizeKnowledgeFileEntry(
|
||||
value: unknown,
|
||||
): KnowledgeFileEntry | undefined {
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
typeof value.id !== "string" ||
|
||||
!value.id.startsWith("kfile-")
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const name =
|
||||
sanitizeFileName(value.name) ??
|
||||
sanitizeFileName(value.originalName) ??
|
||||
"unnamed";
|
||||
const originalName = sanitizeFileName(value.originalName) ?? name;
|
||||
const mimeType =
|
||||
typeof value.mimeType === "string" && value.mimeType.includes("/")
|
||||
? value.mimeType
|
||||
: inferMimeType(originalName);
|
||||
const size =
|
||||
typeof value.size === "number" &&
|
||||
Number.isFinite(value.size) &&
|
||||
value.size > 0
|
||||
? Math.floor(value.size)
|
||||
: 0;
|
||||
const isText = value.isText === true;
|
||||
const extractedText =
|
||||
isText && typeof value.extractedText === "string"
|
||||
? trimExtractedText(value.extractedText)
|
||||
: undefined;
|
||||
const addedAt =
|
||||
typeof value.addedAt === "number" &&
|
||||
Number.isFinite(value.addedAt) &&
|
||||
value.addedAt > 0
|
||||
? Math.floor(value.addedAt)
|
||||
: Date.now();
|
||||
return {
|
||||
id: value.id,
|
||||
name,
|
||||
originalName,
|
||||
mimeType,
|
||||
size,
|
||||
isText,
|
||||
extractedText,
|
||||
extractedTextLength: extractedText?.length ?? 0,
|
||||
addedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export function scoreKnowledgeFile(
|
||||
file: KnowledgeFileEntry,
|
||||
queryTokens: ReadonlySet<string>,
|
||||
normalizedQuery: string,
|
||||
): number {
|
||||
const nameTokens = tokenize(file.name);
|
||||
let overlapCount = 0;
|
||||
for (const token of queryTokens) {
|
||||
if (nameTokens.has(token)) {
|
||||
overlapCount += 1;
|
||||
}
|
||||
}
|
||||
const textTokens = file.extractedText
|
||||
? tokenize(file.extractedText)
|
||||
: new Set<string>();
|
||||
for (const token of queryTokens) {
|
||||
if (textTokens.has(token)) {
|
||||
overlapCount += 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedName = normalizeKnowledgeComparisonValue(file.name);
|
||||
const normalizedText = normalizeKnowledgeComparisonValue(
|
||||
file.extractedText ?? "",
|
||||
);
|
||||
const phraseBonus =
|
||||
normalizedName.includes(normalizedQuery) ||
|
||||
normalizedText.includes(normalizedQuery)
|
||||
? 2
|
||||
: 0;
|
||||
if (overlapCount === 0 && phraseBonus === 0) {
|
||||
return 0;
|
||||
}
|
||||
const recencyBonus =
|
||||
Math.max(0, 1 - (Date.now() - file.addedAt) / (30 * 24 * 60 * 60 * 1000)) *
|
||||
0.3;
|
||||
const sizePenalty =
|
||||
Math.max(0, file.size / maxKnowledgeFileSizeBytes) * 0.2;
|
||||
return overlapCount + phraseBonus + recencyBonus - sizePenalty;
|
||||
}
|
||||
|
||||
export function extractTextContent(data: Buffer): string {
|
||||
const raw = data.toString("utf8");
|
||||
const normalized = raw.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
||||
return trimExtractedText(normalized);
|
||||
}
|
||||
|
||||
export function sanitizeFileName(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const name = value.replace(/\\/g, "/").split("/").pop() ?? "";
|
||||
const sanitized = name
|
||||
.replace(/[\x00-\x1f\x7f<>|:*"?\\]/g, "_")
|
||||
.trim();
|
||||
if (!sanitized || sanitized.length > maxFileNameLength) return undefined;
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
export function trimFileName(name: string): string {
|
||||
if (name.length <= maxFileNameLength) return name;
|
||||
const extension = extname(name);
|
||||
const base = name.slice(0, name.length - extension.length);
|
||||
const maxBaseLength = maxFileNameLength - extension.length - 1;
|
||||
return `${base.slice(0, Math.max(0, maxBaseLength))}…${extension}`;
|
||||
}
|
||||
|
||||
export function inferMimeType(fileName: string): string {
|
||||
const ext = extname(fileName).toLowerCase();
|
||||
switch (ext) {
|
||||
case ".txt":
|
||||
return "text/plain";
|
||||
case ".md":
|
||||
return "text/markdown";
|
||||
case ".json":
|
||||
return "application/json";
|
||||
case ".js":
|
||||
return "text/javascript";
|
||||
case ".ts":
|
||||
return "text/typescript";
|
||||
case ".jsx":
|
||||
return "text/jsx";
|
||||
case ".tsx":
|
||||
return "text/tsx";
|
||||
case ".css":
|
||||
return "text/css";
|
||||
case ".html":
|
||||
return "text/html";
|
||||
case ".xml":
|
||||
return "text/xml";
|
||||
case ".yaml":
|
||||
case ".yml":
|
||||
return "text/yaml";
|
||||
case ".csv":
|
||||
return "text/csv";
|
||||
case ".pdf":
|
||||
return "application/pdf";
|
||||
case ".png":
|
||||
return "image/png";
|
||||
case ".jpg":
|
||||
case ".jpeg":
|
||||
return "image/jpeg";
|
||||
case ".gif":
|
||||
return "image/gif";
|
||||
case ".webp":
|
||||
return "image/webp";
|
||||
case ".svg":
|
||||
return "image/svg+xml";
|
||||
case ".mp3":
|
||||
return "audio/mpeg";
|
||||
case ".mp4":
|
||||
return "video/mp4";
|
||||
default:
|
||||
return "application/octet-stream";
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeKnowledgeQuery(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function normalizeKnowledgeComparisonValue(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function tokenize(value: string): Set<string> {
|
||||
return new Set(
|
||||
normalizeKnowledgeComparisonValue(value)
|
||||
.split(" ")
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length >= 2),
|
||||
);
|
||||
}
|
||||
|
||||
export function isValidUtf8Buffer(data: Buffer): boolean {
|
||||
const bufferConstructor = Buffer as unknown as {
|
||||
isUtf8?(data: Buffer): boolean;
|
||||
};
|
||||
if (typeof bufferConstructor.isUtf8 === "function") {
|
||||
return bufferConstructor.isUtf8(data);
|
||||
}
|
||||
try {
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(data);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function sortKnowledgeFiles(
|
||||
files: readonly KnowledgeFileEntry[],
|
||||
): readonly KnowledgeFileEntry[] {
|
||||
return [...files].sort(
|
||||
(left, right) =>
|
||||
right.addedAt - left.addedAt || left.name.localeCompare(right.name),
|
||||
);
|
||||
}
|
||||
|
||||
function trimExtractedText(text: string): string {
|
||||
const trimmed = text.replace(/[ \t]+/g, " ").trim();
|
||||
if (trimmed.length <= maxExtractedTextChars) {
|
||||
return trimmed;
|
||||
}
|
||||
return `${trimmed.slice(0, maxExtractedTextChars)}\n…`;
|
||||
}
|
||||
|
||||
function extname(name: string): string {
|
||||
const dotIndex = name.lastIndexOf(".");
|
||||
return dotIndex <= 0 ? "" : name.slice(dotIndex);
|
||||
}
|
||||
228
apps/desktop/src/knowledge-store-core-support.ts
Normal file
228
apps/desktop/src/knowledge-store-core-support.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
import { Buffer } from "node:buffer";
|
||||
import { mkdir, rename, writeFile } from "node:fs/promises";
|
||||
import { dirname, extname } from "node:path";
|
||||
|
||||
import {
|
||||
extractTextContent,
|
||||
inferMimeType,
|
||||
isRecord,
|
||||
isValidUtf8Buffer,
|
||||
maxKnowledgeFileSizeBytes,
|
||||
normalizeKnowledgeComparisonValue,
|
||||
normalizeKnowledgeFileEntry as parseKnowledgeFileEntry,
|
||||
normalizeKnowledgeQuery,
|
||||
sanitizeFileName,
|
||||
scoreKnowledgeFile,
|
||||
sortKnowledgeFiles,
|
||||
tokenize,
|
||||
trimFileName,
|
||||
} from "./knowledge-store-core-helpers.js";
|
||||
|
||||
export interface KnowledgeFileEntry {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly originalName: string;
|
||||
readonly mimeType: string;
|
||||
readonly size: number;
|
||||
readonly isText: boolean;
|
||||
readonly extractedText: string | undefined;
|
||||
readonly extractedTextLength: number;
|
||||
readonly addedAt: number;
|
||||
}
|
||||
|
||||
export interface KnowledgeFileSearchHit {
|
||||
readonly file: KnowledgeFileEntry;
|
||||
readonly score: number;
|
||||
}
|
||||
|
||||
export interface StoreKnowledgeFileInput {
|
||||
readonly name: string;
|
||||
readonly data: Buffer;
|
||||
readonly mimeType?: string;
|
||||
}
|
||||
|
||||
export interface KnowledgeStoreIndexV1 {
|
||||
readonly version: 1;
|
||||
readonly files: readonly KnowledgeFileEntry[];
|
||||
}
|
||||
|
||||
export const knowledgeStoreVersion = 1;
|
||||
|
||||
const maxKnowledgeFiles = 100;
|
||||
const maxRelevantKnowledgeItems = 4;
|
||||
const maxRelevantKnowledgeChars = 2_000;
|
||||
|
||||
export function clampKnowledgeLimit(value: number, fallback: number): number {
|
||||
if (!Number.isFinite(value)) return fallback;
|
||||
return Math.min(Math.max(Math.round(value), 1), 100);
|
||||
}
|
||||
|
||||
export function cloneKnowledgeFileEntry(
|
||||
entry: KnowledgeFileEntry,
|
||||
): KnowledgeFileEntry {
|
||||
return { ...entry };
|
||||
}
|
||||
|
||||
export function createKnowledgeFileEntry(
|
||||
id: string,
|
||||
input: StoreKnowledgeFileInput,
|
||||
addedAt = Date.now(),
|
||||
): { readonly entry: KnowledgeFileEntry; readonly extension: string } {
|
||||
const originalName = sanitizeFileName(input.name);
|
||||
if (!originalName) {
|
||||
throw new Error("File name is required.");
|
||||
}
|
||||
if (!Buffer.isBuffer(input.data)) {
|
||||
throw new Error("File data must be a Buffer.");
|
||||
}
|
||||
if (input.data.length === 0) {
|
||||
throw new Error("File cannot be empty.");
|
||||
}
|
||||
if (input.data.length > maxKnowledgeFileSizeBytes) {
|
||||
throw new Error(
|
||||
`File is too large. Maximum size is ${formatBytes(maxKnowledgeFileSizeBytes)}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const displayName = trimFileName(originalName);
|
||||
const extension = extname(originalName).toLowerCase();
|
||||
const mimeType =
|
||||
input.mimeType && input.mimeType.includes("/")
|
||||
? input.mimeType.trim()
|
||||
: inferMimeType(originalName);
|
||||
const isText = isValidUtf8Buffer(input.data) && !input.data.includes(0);
|
||||
const extractedText = isText ? extractTextContent(input.data) : undefined;
|
||||
|
||||
return {
|
||||
extension,
|
||||
entry: {
|
||||
id,
|
||||
name: displayName,
|
||||
originalName,
|
||||
mimeType,
|
||||
size: input.data.length,
|
||||
isText,
|
||||
extractedText,
|
||||
extractedTextLength: extractedText?.length ?? 0,
|
||||
addedAt,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function searchKnowledgeFiles(
|
||||
files: readonly KnowledgeFileEntry[],
|
||||
query: string,
|
||||
limit = 10,
|
||||
): readonly KnowledgeFileSearchHit[] {
|
||||
const trimmedQuery = normalizeKnowledgeQuery(query);
|
||||
const nextLimit = clampKnowledgeLimit(limit, 10);
|
||||
if (!trimmedQuery) {
|
||||
return sortKnowledgeFiles(files)
|
||||
.slice(0, nextLimit)
|
||||
.map((file) => ({ file: cloneKnowledgeFileEntry(file), score: 0 }));
|
||||
}
|
||||
|
||||
const queryTokens = tokenize(trimmedQuery);
|
||||
const normalizedQuery = normalizeKnowledgeComparisonValue(trimmedQuery);
|
||||
return files
|
||||
.map((file) => ({
|
||||
file,
|
||||
score: scoreKnowledgeFile(file, queryTokens, normalizedQuery),
|
||||
}))
|
||||
.filter((hit) => hit.score > 0)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.score - left.score ||
|
||||
right.file.addedAt - left.file.addedAt ||
|
||||
left.file.name.localeCompare(right.file.name),
|
||||
)
|
||||
.slice(0, nextLimit)
|
||||
.map((hit) => ({
|
||||
file: cloneKnowledgeFileEntry(hit.file),
|
||||
score: Math.round(hit.score * 100) / 100,
|
||||
}));
|
||||
}
|
||||
|
||||
export function buildRelevantKnowledgeContext(
|
||||
files: readonly KnowledgeFileEntry[],
|
||||
prompt: string,
|
||||
): string | undefined {
|
||||
const trimmedQuery = normalizeKnowledgeQuery(prompt);
|
||||
if (!trimmedQuery) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const hits = searchKnowledgeFiles(
|
||||
files,
|
||||
trimmedQuery,
|
||||
maxRelevantKnowledgeItems,
|
||||
).filter((hit) => hit.score > 0);
|
||||
if (hits.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
let usedChars = 0;
|
||||
for (const hit of hits) {
|
||||
const file = hit.file;
|
||||
const header = `- [file: ${file.name}] (${formatBytes(file.size)}, ${file.mimeType})`;
|
||||
const body =
|
||||
file.isText && file.extractedText
|
||||
? file.extractedText
|
||||
: "[Non-text file: content not shown.]";
|
||||
const entry = `${header}\n${body}`;
|
||||
if (usedChars > 0 && usedChars + entry.length + 2 > maxRelevantKnowledgeChars) {
|
||||
break;
|
||||
}
|
||||
lines.push(entry);
|
||||
usedChars += entry.length + 2;
|
||||
}
|
||||
|
||||
if (lines.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return [
|
||||
"Relevant stored files:",
|
||||
...lines,
|
||||
"Use the above files only when they help answer the current request.",
|
||||
].join("\n\n");
|
||||
}
|
||||
|
||||
export function normalizeKnowledgeStore(value: unknown): KnowledgeStoreIndexV1 {
|
||||
const record = isRecord(value) ? value : {};
|
||||
const files = Array.isArray(record.files)
|
||||
? record.files
|
||||
.map((file) => parseKnowledgeFileEntry(file))
|
||||
.filter((file): file is KnowledgeFileEntry => Boolean(file))
|
||||
: [];
|
||||
return {
|
||||
version: 1,
|
||||
files: pruneKnowledgeFiles(files),
|
||||
};
|
||||
}
|
||||
|
||||
export function pruneKnowledgeFiles(
|
||||
files: readonly KnowledgeFileEntry[],
|
||||
): readonly KnowledgeFileEntry[] {
|
||||
if (files.length <= maxKnowledgeFiles) {
|
||||
return [...files];
|
||||
}
|
||||
return sortKnowledgeFiles(files).slice(0, maxKnowledgeFiles);
|
||||
}
|
||||
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
export async function atomicWrite(
|
||||
targetPath: string,
|
||||
data: Buffer,
|
||||
): Promise<void> {
|
||||
await mkdir(dirname(targetPath), { recursive: true });
|
||||
const tempPath = `${targetPath}.${process.pid}.tmp`;
|
||||
await writeFile(tempPath, data);
|
||||
await rename(tempPath, targetPath);
|
||||
}
|
||||
|
|
@ -1,48 +1,34 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { mkdir, rename, unlink, writeFile } from "node:fs/promises";
|
||||
import { Buffer } from "node:buffer";
|
||||
import { TextDecoder } from "node:util";
|
||||
import { unlink } from "node:fs/promises";
|
||||
import { dirname, extname, join } from "node:path";
|
||||
|
||||
export interface KnowledgeFileEntry {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly originalName: string;
|
||||
readonly mimeType: string;
|
||||
readonly size: number;
|
||||
readonly isText: boolean;
|
||||
readonly extractedText: string | undefined;
|
||||
readonly extractedTextLength: number;
|
||||
readonly addedAt: number;
|
||||
}
|
||||
import {
|
||||
atomicWrite,
|
||||
buildRelevantKnowledgeContext,
|
||||
clampKnowledgeLimit,
|
||||
cloneKnowledgeFileEntry,
|
||||
createKnowledgeFileEntry,
|
||||
knowledgeStoreVersion,
|
||||
normalizeKnowledgeStore,
|
||||
pruneKnowledgeFiles,
|
||||
searchKnowledgeFiles,
|
||||
type KnowledgeFileEntry,
|
||||
type KnowledgeFileSearchHit,
|
||||
type KnowledgeStoreIndexV1,
|
||||
type StoreKnowledgeFileInput,
|
||||
} from "./knowledge-store-core-support.js";
|
||||
|
||||
export interface KnowledgeFileSearchHit {
|
||||
readonly file: KnowledgeFileEntry;
|
||||
readonly score: number;
|
||||
}
|
||||
export { formatBytes } from "./knowledge-store-core-support.js";
|
||||
export type {
|
||||
KnowledgeFileEntry,
|
||||
KnowledgeFileSearchHit,
|
||||
StoreKnowledgeFileInput,
|
||||
} from "./knowledge-store-core-support.js";
|
||||
|
||||
export interface StoreKnowledgeFileInput {
|
||||
readonly name: string;
|
||||
readonly data: Buffer;
|
||||
readonly mimeType?: string;
|
||||
}
|
||||
|
||||
const knowledgeStoreVersion = 1;
|
||||
const maxKnowledgeFiles = 100;
|
||||
const maxKnowledgeFileSizeBytes = 5 * 1024 * 1024;
|
||||
const maxExtractedTextChars = 2_000;
|
||||
const maxRelevantKnowledgeItems = 4;
|
||||
const maxRelevantKnowledgeChars = 2_000;
|
||||
const maxFileNameLength = 128;
|
||||
const knowledgeFilesDirName = "files";
|
||||
const knowledgeStoreIndexFileName = "knowledge-store.json";
|
||||
|
||||
interface KnowledgeStoreIndexV1 {
|
||||
readonly version: 1;
|
||||
readonly files: readonly KnowledgeFileEntry[];
|
||||
}
|
||||
|
||||
export class KnowledgeStore {
|
||||
private cachedStore: KnowledgeStoreIndexV1 | null = null;
|
||||
|
||||
|
|
@ -51,68 +37,23 @@ export class KnowledgeStore {
|
|||
listFiles(limit = 50): readonly KnowledgeFileEntry[] {
|
||||
const nextLimit = clampKnowledgeLimit(limit, 50);
|
||||
return [...this.getStore().files]
|
||||
.sort((left, right) => right.addedAt - left.addedAt || left.name.localeCompare(right.name))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.addedAt - left.addedAt || left.name.localeCompare(right.name),
|
||||
)
|
||||
.slice(0, nextLimit)
|
||||
.map(cloneKnowledgeFileEntry);
|
||||
}
|
||||
|
||||
searchFiles(query: string, limit = 10): readonly KnowledgeFileSearchHit[] {
|
||||
const trimmedQuery = normalizeKnowledgeQuery(query);
|
||||
const nextLimit = clampKnowledgeLimit(limit, 10);
|
||||
if (!trimmedQuery) {
|
||||
return this.listFiles(nextLimit).map((file) => ({ file, score: 0 }));
|
||||
}
|
||||
|
||||
const queryTokens = tokenize(trimmedQuery);
|
||||
const normalizedQuery = normalizeKnowledgeComparisonValue(trimmedQuery);
|
||||
return this.getStore().files
|
||||
.map((file) => ({ file, score: scoreKnowledgeFile(file, queryTokens, normalizedQuery) }))
|
||||
.filter((hit) => hit.score > 0)
|
||||
.sort((left, right) => right.score - left.score || right.file.addedAt - left.file.addedAt || left.file.name.localeCompare(right.file.name))
|
||||
.slice(0, nextLimit)
|
||||
.map((hit) => ({ file: cloneKnowledgeFileEntry(hit.file), score: Math.round(hit.score * 100) / 100 }));
|
||||
return searchKnowledgeFiles(this.getStore().files, query, limit);
|
||||
}
|
||||
|
||||
async storeFile(input: StoreKnowledgeFileInput): Promise<KnowledgeFileEntry> {
|
||||
const originalName = sanitizeFileName(input.name);
|
||||
if (!originalName) {
|
||||
throw new Error("File name is required.");
|
||||
}
|
||||
if (!Buffer.isBuffer(input.data)) {
|
||||
throw new Error("File data must be a Buffer.");
|
||||
}
|
||||
if (input.data.length === 0) {
|
||||
throw new Error("File cannot be empty.");
|
||||
}
|
||||
if (input.data.length > maxKnowledgeFileSizeBytes) {
|
||||
throw new Error(`File is too large. Maximum size is ${formatBytes(maxKnowledgeFileSizeBytes)}.`);
|
||||
}
|
||||
|
||||
const id = `kfile-${randomUUID()}`;
|
||||
const displayName = trimFileName(originalName);
|
||||
const extension = extname(originalName).toLowerCase();
|
||||
const mimeType = input.mimeType && input.mimeType.includes("/")
|
||||
? input.mimeType.trim()
|
||||
: inferMimeType(originalName);
|
||||
|
||||
const isText = isValidUtf8Buffer(input.data) && !input.data.includes(0);
|
||||
const extractedText = isText ? extractTextContent(input.data) : undefined;
|
||||
|
||||
const entry: KnowledgeFileEntry = {
|
||||
id,
|
||||
name: displayName,
|
||||
originalName,
|
||||
mimeType,
|
||||
size: input.data.length,
|
||||
isText,
|
||||
extractedText,
|
||||
extractedTextLength: extractedText?.length ?? 0,
|
||||
addedAt: Date.now(),
|
||||
};
|
||||
|
||||
const { entry, extension } = createKnowledgeFileEntry(id, input);
|
||||
const store = this.getStore();
|
||||
const filesDir = this.getFilesDir();
|
||||
await mkdir(filesDir, { recursive: true });
|
||||
|
||||
const filePath = join(filesDir, `${id}${extension}`);
|
||||
await atomicWrite(filePath, input.data);
|
||||
|
|
@ -154,38 +95,7 @@ export class KnowledgeStore {
|
|||
}
|
||||
|
||||
buildRelevantContext(prompt: string): string | undefined {
|
||||
const trimmedQuery = normalizeKnowledgeQuery(prompt);
|
||||
if (!trimmedQuery) {
|
||||
return undefined;
|
||||
}
|
||||
const hits = this.searchFiles(trimmedQuery, maxRelevantKnowledgeItems).filter((hit) => hit.score > 0);
|
||||
if (hits.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
let usedChars = 0;
|
||||
for (const hit of hits) {
|
||||
const file = hit.file;
|
||||
const header = `- [file: ${file.name}] (${formatBytes(file.size)}, ${file.mimeType})`;
|
||||
const body = file.isText && file.extractedText ? file.extractedText : "[Non-text file: content not shown.]";
|
||||
const entry = `${header}\n${body}`;
|
||||
if (usedChars > 0 && usedChars + entry.length + 2 > maxRelevantKnowledgeChars) {
|
||||
break;
|
||||
}
|
||||
lines.push(entry);
|
||||
usedChars += entry.length + 2;
|
||||
}
|
||||
|
||||
if (lines.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return [
|
||||
"Relevant stored files:",
|
||||
...lines,
|
||||
"Use the above files only when they help answer the current request.",
|
||||
].join("\n\n");
|
||||
return buildRelevantKnowledgeContext(this.getStore().files, prompt);
|
||||
}
|
||||
|
||||
getFilePath(id: string): string | null {
|
||||
|
|
@ -205,14 +115,14 @@ export class KnowledgeStore {
|
|||
const raw = JSON.parse(readFileSync(this.getIndexPath(), "utf8")) as unknown;
|
||||
this.cachedStore = normalizeKnowledgeStore(raw);
|
||||
} catch {
|
||||
this.cachedStore = { version: 1, files: [] };
|
||||
this.cachedStore = { version: knowledgeStoreVersion, files: [] };
|
||||
}
|
||||
return this.cachedStore;
|
||||
}
|
||||
|
||||
private commitStore(store: KnowledgeStoreIndexV1): void {
|
||||
this.cachedStore = {
|
||||
version: 1,
|
||||
version: knowledgeStoreVersion,
|
||||
files: store.files.map(cloneKnowledgeFileEntry),
|
||||
};
|
||||
const path = this.getIndexPath();
|
||||
|
|
@ -231,188 +141,3 @@ export class KnowledgeStore {
|
|||
return join(this.baseDir, knowledgeStoreIndexFileName);
|
||||
}
|
||||
}
|
||||
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
function normalizeKnowledgeStore(value: unknown): KnowledgeStoreIndexV1 {
|
||||
const record = isRecord(value) ? value : {};
|
||||
const files = Array.isArray(record.files)
|
||||
? record.files.map((file) => normalizeKnowledgeFileEntry(file)).filter((file): file is KnowledgeFileEntry => Boolean(file))
|
||||
: [];
|
||||
return {
|
||||
version: 1,
|
||||
files: pruneKnowledgeFiles(files),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeKnowledgeFileEntry(value: unknown): KnowledgeFileEntry | undefined {
|
||||
if (!isRecord(value) || typeof value.id !== "string" || !value.id.startsWith("kfile-")) {
|
||||
return undefined;
|
||||
}
|
||||
const name = sanitizeFileName(value.name) ?? sanitizeFileName(value.originalName) ?? "unnamed";
|
||||
const originalName = sanitizeFileName(value.originalName) ?? name;
|
||||
const mimeType = typeof value.mimeType === "string" && value.mimeType.includes("/") ? value.mimeType : inferMimeType(originalName);
|
||||
const size = typeof value.size === "number" && Number.isFinite(value.size) && value.size > 0 ? Math.floor(value.size) : 0;
|
||||
const isText = value.isText === true;
|
||||
const extractedText = isText && typeof value.extractedText === "string" ? trimExtractedText(value.extractedText) : undefined;
|
||||
const addedAt = typeof value.addedAt === "number" && Number.isFinite(value.addedAt) && value.addedAt > 0 ? Math.floor(value.addedAt) : Date.now();
|
||||
return {
|
||||
id: value.id,
|
||||
name,
|
||||
originalName,
|
||||
mimeType,
|
||||
size,
|
||||
isText,
|
||||
extractedText,
|
||||
extractedTextLength: extractedText?.length ?? 0,
|
||||
addedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function pruneKnowledgeFiles(files: readonly KnowledgeFileEntry[]): readonly KnowledgeFileEntry[] {
|
||||
if (files.length <= maxKnowledgeFiles) {
|
||||
return [...files];
|
||||
}
|
||||
return [...files]
|
||||
.sort((left, right) => right.addedAt - left.addedAt)
|
||||
.slice(0, maxKnowledgeFiles);
|
||||
}
|
||||
|
||||
function scoreKnowledgeFile(file: KnowledgeFileEntry, queryTokens: ReadonlySet<string>, normalizedQuery: string): number {
|
||||
const nameTokens = tokenize(file.name);
|
||||
let overlapCount = 0;
|
||||
for (const token of queryTokens) {
|
||||
if (nameTokens.has(token)) {
|
||||
overlapCount += 1;
|
||||
}
|
||||
}
|
||||
const textTokens = file.extractedText ? tokenize(file.extractedText) : new Set<string>();
|
||||
for (const token of queryTokens) {
|
||||
if (textTokens.has(token)) {
|
||||
overlapCount += 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedName = normalizeKnowledgeComparisonValue(file.name);
|
||||
const normalizedText = normalizeKnowledgeComparisonValue(file.extractedText ?? "");
|
||||
const phraseBonus = normalizedName.includes(normalizedQuery) || normalizedText.includes(normalizedQuery) ? 2 : 0;
|
||||
if (overlapCount === 0 && phraseBonus === 0) {
|
||||
return 0;
|
||||
}
|
||||
const recencyBonus = Math.max(0, 1 - (Date.now() - file.addedAt) / (30 * 24 * 60 * 60 * 1000)) * 0.3;
|
||||
const sizePenalty = Math.max(0, file.size / maxKnowledgeFileSizeBytes) * 0.2;
|
||||
return overlapCount + phraseBonus + recencyBonus - sizePenalty;
|
||||
}
|
||||
|
||||
function extractTextContent(data: Buffer): string {
|
||||
const raw = data.toString("utf8");
|
||||
const normalized = raw.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
||||
return trimExtractedText(normalized);
|
||||
}
|
||||
|
||||
function trimExtractedText(text: string): string {
|
||||
const trimmed = text.replace(/[ \t]+/g, " ").trim();
|
||||
if (trimmed.length <= maxExtractedTextChars) {
|
||||
return trimmed;
|
||||
}
|
||||
return `${trimmed.slice(0, maxExtractedTextChars)}\n…`;
|
||||
}
|
||||
|
||||
function sanitizeFileName(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const name = value.replace(/\\/g, "/").split("/").pop() ?? "";
|
||||
const sanitized = name.replace(/[\x00-\x1f\x7f<>|:*"?\\]/g, "_").trim();
|
||||
if (!sanitized || sanitized.length > maxFileNameLength) return undefined;
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
function trimFileName(name: string): string {
|
||||
if (name.length <= maxFileNameLength) return name;
|
||||
const extension = extname(name);
|
||||
const base = name.slice(0, name.length - extension.length);
|
||||
const maxBaseLength = maxFileNameLength - extension.length - 1;
|
||||
return `${base.slice(0, Math.max(0, maxBaseLength))}…${extension}`;
|
||||
}
|
||||
|
||||
function inferMimeType(fileName: string): string {
|
||||
const ext = extname(fileName).toLowerCase();
|
||||
switch (ext) {
|
||||
case ".txt": return "text/plain";
|
||||
case ".md": return "text/markdown";
|
||||
case ".json": return "application/json";
|
||||
case ".js": return "text/javascript";
|
||||
case ".ts": return "text/typescript";
|
||||
case ".jsx": return "text/jsx";
|
||||
case ".tsx": return "text/tsx";
|
||||
case ".css": return "text/css";
|
||||
case ".html": return "text/html";
|
||||
case ".xml": return "text/xml";
|
||||
case ".yaml":
|
||||
case ".yml": return "text/yaml";
|
||||
case ".csv": return "text/csv";
|
||||
case ".pdf": return "application/pdf";
|
||||
case ".png": return "image/png";
|
||||
case ".jpg":
|
||||
case ".jpeg": return "image/jpeg";
|
||||
case ".gif": return "image/gif";
|
||||
case ".webp": return "image/webp";
|
||||
case ".svg": return "image/svg+xml";
|
||||
case ".mp3": return "audio/mpeg";
|
||||
case ".mp4": return "video/mp4";
|
||||
default: return "application/octet-stream";
|
||||
}
|
||||
}
|
||||
|
||||
function cloneKnowledgeFileEntry(entry: KnowledgeFileEntry): KnowledgeFileEntry {
|
||||
return { ...entry };
|
||||
}
|
||||
|
||||
function clampKnowledgeLimit(value: number, fallback: number): number {
|
||||
if (!Number.isFinite(value)) return fallback;
|
||||
return Math.min(Math.max(Math.round(value), 1), 100);
|
||||
}
|
||||
|
||||
function normalizeKnowledgeQuery(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
||||
}
|
||||
|
||||
function normalizeKnowledgeComparisonValue(value: string): string {
|
||||
return value.toLowerCase().replace(/[^a-z0-9\s]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function tokenize(value: string): Set<string> {
|
||||
return new Set(
|
||||
normalizeKnowledgeComparisonValue(value)
|
||||
.split(" ")
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length >= 2),
|
||||
);
|
||||
}
|
||||
|
||||
function isValidUtf8Buffer(data: Buffer): boolean {
|
||||
const bufferConstructor = Buffer as unknown as { isUtf8?(data: Buffer): boolean };
|
||||
if (typeof bufferConstructor.isUtf8 === "function") {
|
||||
return bufferConstructor.isUtf8(data);
|
||||
}
|
||||
try {
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(data);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
async function atomicWrite(targetPath: string, data: Buffer): Promise<void> {
|
||||
await mkdir(dirname(targetPath), { recursive: true });
|
||||
const tempPath = `${targetPath}.${process.pid}.tmp`;
|
||||
await writeFile(tempPath, data);
|
||||
await rename(tempPath, targetPath);
|
||||
}
|
||||
|
|
|
|||
63
apps/desktop/tests/knowledge-store-core-seam.test.ts
Normal file
63
apps/desktop/tests/knowledge-store-core-seam.test.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const desktopRoot =
|
||||
process.env.FAMILIAROS_DESKTOP_ROOT ??
|
||||
resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const knowledgeStoreCoreSource = readFileSync(
|
||||
resolve(desktopRoot, "src/knowledge-store-core.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const knowledgeStoreCoreSupportSource = readFileSync(
|
||||
resolve(desktopRoot, "src/knowledge-store-core-support.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const knowledgeStoreCoreHelpersSource = readFileSync(
|
||||
resolve(desktopRoot, "src/knowledge-store-core-helpers.ts"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
assert.match(
|
||||
knowledgeStoreCoreSource,
|
||||
/from "\.\/knowledge-store-core-support(?:\.js)?"/,
|
||||
"knowledge-store core must import the extracted support seam.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreSource,
|
||||
/export class KnowledgeStore/,
|
||||
"knowledge-store core must keep exporting the public KnowledgeStore class.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreSupportSource,
|
||||
/export function createKnowledgeFileEntry/,
|
||||
"knowledge-store support must own stored-file validation and shaping.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreSupportSource,
|
||||
/from "\.\/knowledge-store-core-helpers(?:\.js)?"/,
|
||||
"knowledge-store support must import the extracted helper seam.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreSupportSource,
|
||||
/export function searchKnowledgeFiles/,
|
||||
"knowledge-store support must own file search scoring.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreSupportSource,
|
||||
/export function buildRelevantKnowledgeContext/,
|
||||
"knowledge-store support must own relevant-context assembly.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreHelpersSource,
|
||||
/export function normalizeKnowledgeFileEntry/,
|
||||
"knowledge-store helpers must own index-entry normalization.",
|
||||
);
|
||||
assert.match(
|
||||
knowledgeStoreCoreHelpersSource,
|
||||
/export function scoreKnowledgeFile/,
|
||||
"knowledge-store helpers must own low-level search scoring.",
|
||||
);
|
||||
|
||||
console.error("Knowledge store core seam validation passed.");
|
||||
Loading…
Add table
Reference in a new issue