scriptoriumadmin/supermemory
1
0
Fork 0
mirror of https://github.com/supermemoryai/supermemory.git synced 2026-09-06 08:16:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update: catch up with main

Browse source
This commit is contained in:
Dhravya 2024-04-03 00:14:07 -07:00
commit 4ff97a0904
19 changed files with 375 additions and 5909 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
apps/cf-ai-backend/src/env.d.ts vendored Normal file
View file

@ -0,0 +1,7 @@
interface Env {
VECTORIZE_INDEX: VectorizeIndex;
AI: Fetcher;
SECURITY_KEY: string;
OPENAI_API_KEY: string;
GOOGLE_AI_API_KEY: string;
}

141
apps/cf-ai-backend/src/index.ts
View file

@ -7,29 +7,20 @@ import type {
import {
CloudflareVectorizeStore,
} from "@langchain/cloudflare";
import { Ai } from '@cloudflare/ai';
import { OpenAIEmbeddings } from "./OpenAIEmbedder";
import { AiTextGenerationOutput } from "@cloudflare/ai/dist/ai/tasks/text-generation";
export interface Env {
VECTORIZE_INDEX: VectorizeIndex;
AI: Fetcher;
SECURITY_KEY: string;
OPENAI_API_KEY: string;
}
import { GoogleGenerativeAI } from "@google/generative-ai";
import routeMap from "./routes";
function isAuthorized(request: Request, env: Env): boolean {
return request.headers.get('X-Custom-Auth-Key') === env.SECURITY_KEY;
}
export default {
async fetch(request: Request, env: Env) {
async fetch(request: Request, env: Env, ctx: ExecutionContext) {
if (!isAuthorized(request, env)) {
return new Response('Unauthorized', { status: 401 });
}
const pathname = new URL(request.url).pathname;
const embeddings = new OpenAIEmbeddings({
apiKey: env.OPENAI_API_KEY,
modelName: 'text-embedding-3-small',
@ -38,128 +29,26 @@ export default {
const store = new CloudflareVectorizeStore(embeddings, {
index: env.VECTORIZE_INDEX,
});
const ai = new Ai(env.AI)
if (pathname === "/add" && request.method === "POST") {
const genAI = new GoogleGenerativeAI(env.GOOGLE_AI_API_KEY);
const body = await request.json() as {
pageContent: string,
title?: string,
description?: string,
url: string,
user: string
};
const model = genAI.getGenerativeModel({ model: "gemini-pro" });
const url = new URL(request.url);
const path = url.pathname;
const method = request.method.toUpperCase();
if (!body.pageContent || !body.url) {
return new Response(JSON.stringify({ message: "Invalid Page Content" }), { status: 400 });
}
const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${body.pageContent}`
const routeHandlers = routeMap.get(path);
await store.addDocuments([
{
pageContent: newPageContent,
metadata: {
title: body.title ?? "",
description: body.description ?? "",
url: body.url,
user: body.user,
},
},
], {
ids: [`${body.url}`]
})
return new Response(JSON.stringify({ message: "Document Added" }), { status: 200 });
if (!routeHandlers) {
return new Response('Not Found', { status: 404 });
}
else if (pathname === "/query" && request.method === "GET") {
const queryparams = new URL(request.url).searchParams;
const query = queryparams.get("q");
const topK = parseInt(queryparams.get("topK") ?? "5");
const user = queryparams.get("user")
const handler = routeHandlers[method];
const sourcesOnly = (queryparams.get("sourcesOnly") ?? "false")
if (!user) {
return new Response(JSON.stringify({ message: "Invalid User" }), { status: 400 });
}
if (!query) {
return new Response(JSON.stringify({ message: "Invalid Query" }), { status: 400 });
}
const filter: VectorizeVectorMetadataFilter = {
user: {
$eq: user
}
}
const queryAsVector = await embeddings.embedQuery(query);
const resp = await env.VECTORIZE_INDEX.query(queryAsVector, {
topK,
filter
});
if (resp.count === 0) {
return new Response(JSON.stringify({ message: "No Results Found" }), { status: 400 });
}
const highScoreIds = resp.matches.filter(({ score }) => score > 0.3).map(({ id }) => id)
if (sourcesOnly === "true") {
return new Response(JSON.stringify({ ids: highScoreIds }), { status: 200 });
}
const vec = await env.VECTORIZE_INDEX.getByIds(highScoreIds)
if (vec.length === 0 || !vec[0].metadata) {
return new Response(JSON.stringify({ message: "No Results Found" }), { status: 400 });
}
const metadatas = vec.map(({ metadata }) => metadata)
console.log(metadatas)
// TODO: TAKE ALL THE HIGH SCORED IDS INTO CONSIDERATION
const output: AiTextGenerationOutput = await ai.run('@hf/thebloke/mistral-7b-instruct-v0.1-awq', {
prompt: `You are an agent that summarizes a page based on the query. Be direct and concise, don't say 'based on the context'.\n\n Context:\n${vec[0].metadata!.text} \nAnswer this question based on the context. Question: ${query}\nAnswer:`,
stream: true
}) as ReadableStream
return new Response(output, {
headers: {
"content-type": "text/event-stream",
},
});
if (!handler) {
return new Response('Method Not Allowed', { status: 405 });
}
else if (pathname === "/ask" && request.method === "POST") {
const body = await request.json() as {
query: string
};
if (!body.query) {
return new Response(JSON.stringify({ message: "Invalid Page Content" }), { status: 400 });
}
const output: AiTextGenerationOutput = await ai.run('@hf/thebloke/mistral-7b-instruct-v0.1-awq', {
prompt: `You are an agent that answers a question based on the query. Be direct and concise, don't say 'based on the context'.\n\n Context:\n${body.query} \nAnswer this question based on the context. Question: ${body.query}\nAnswer:`,
stream: true
}) as ReadableStream
return new Response(output, {
headers: {
"content-type": "text/event-stream",
},
});
}
return new Response(JSON.stringify({ message: "Invalid Request" }), { status: 400 });
return await handler(request, store, embeddings, model, env, ctx);
},
};

23
apps/cf-ai-backend/src/routes.ts Normal file
View file

@ -0,0 +1,23 @@
import { CloudflareVectorizeStore } from '@langchain/cloudflare';
import * as apiAdd from './routes/add';
import * as apiQuery from "./routes/query"
import * as apiAsk from "./routes/ask"
import { OpenAIEmbeddings } from './OpenAIEmbedder';
import { GenerativeModel } from '@google/generative-ai';
import { Request } from '@cloudflare/workers-types';
type RouteHandler = (request: Request, store: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env: Env, ctx?: ExecutionContext) => Promise<Response>;
const routeMap = new Map<string, Record<string, RouteHandler>>();
routeMap.set('/add', apiAdd);
routeMap.set('/query', apiQuery);
routeMap.set('/ask', apiAsk);
// Add more route mappings as needed
// routeMap.set('/api/otherRoute', { ... });
export default routeMap;

36
apps/cf-ai-backend/src/routes/add.ts Normal file
View file

@ -0,0 +1,36 @@
import { Request } from "@cloudflare/workers-types";
import { type CloudflareVectorizeStore } from "@langchain/cloudflare";
export async function POST(request: Request, store: CloudflareVectorizeStore) {
const body = await request.json() as {
pageContent: string,
title?: string,
description?: string,
space?: string,
url: string,
user: string
};
if (!body.pageContent || !body.url) {
return new Response(JSON.stringify({ message: "Invalid Page Content" }), { status: 400 });
}
const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${body.pageContent}`
await store.addDocuments([
{
pageContent: newPageContent,
metadata: {
title: body.title ?? "",
description: body.description ?? "",
space: body.space ?? "",
url: body.url,
user: body.user,
},
},
], {
ids: [`${body.url}`]
})
return new Response(JSON.stringify({ message: "Document Added" }), { status: 200 });
}

35
apps/cf-ai-backend/src/routes/ask.ts Normal file
View file

@ -0,0 +1,35 @@
import { GenerativeModel } from "@google/generative-ai";
import { OpenAIEmbeddings } from "../OpenAIEmbedder";
import { CloudflareVectorizeStore } from "@langchain/cloudflare";
import { Request } from "@cloudflare/workers-types";
export async function POST(request: Request, _: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env?: Env) {
const body = await request.json() as {
query: string
};
if (!body.query) {
return new Response(JSON.stringify({ message: "Invalid Page Content" }), { status: 400 });
}
const prompt = `You are an agent that answers a question based on the query. don't say 'based on the context'.\n\n Context:\n${body.query} \nAnswer this question based on the context. Question: ${body.query}\nAnswer:`
const output = await model.generateContentStream(prompt);
const response = new Response(
new ReadableStream({
async start(controller) {
const converter = new TextEncoder();
for await (const chunk of output.stream) {
const chunkText = await chunk.text();
console.log(chunkText);
const encodedChunk = converter.encode("data: " + JSON.stringify({ "response": chunkText }) + "\n\n");
controller.enqueue(encodedChunk);
}
const doneChunk = converter.encode("data: [DONE]");
controller.enqueue(doneChunk);
controller.close();
}
})
);
return response;
}

79
apps/cf-ai-backend/src/routes/query.ts Normal file
View file

@ -0,0 +1,79 @@
import { GenerativeModel } from "@google/generative-ai";
import { OpenAIEmbeddings } from "../OpenAIEmbedder";
import { CloudflareVectorizeStore } from "@langchain/cloudflare";
import { Request } from "@cloudflare/workers-types";
export async function GET(request: Request, _: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env?: Env) {
const queryparams = new URL(request.url).searchParams;
const query = queryparams.get("q");
const topK = parseInt(queryparams.get("topK") ?? "5");
const user = queryparams.get("user")
const space = queryparams.get("space")
const sourcesOnly = (queryparams.get("sourcesOnly") ?? "false")
if (!user) {
return new Response(JSON.stringify({ message: "Invalid User" }), { status: 400 });
}
if (!query) {
return new Response(JSON.stringify({ message: "Invalid Query" }), { status: 400 });
}
const filter: VectorizeVectorMetadataFilter = {
user: {
$eq: user
}
}
if (space) {
filter.space = {
$eq: space
}
}
const queryAsVector = await embeddings.embedQuery(query);
const resp = await env!.VECTORIZE_INDEX.query(queryAsVector, {
topK,
filter
});
if (resp.count === 0) {
return new Response(JSON.stringify({ message: "No Results Found" }), { status: 400 });
}
const highScoreIds = resp.matches.filter(({ score }) => score > 0.3).map(({ id }) => id)
if (sourcesOnly === "true") {
return new Response(JSON.stringify({ ids: highScoreIds }), { status: 200 });
}
const vec = await env!.VECTORIZE_INDEX.getByIds(highScoreIds)
if (vec.length === 0 || !vec[0].metadata) {
return new Response(JSON.stringify({ message: "No Results Found" }), { status: 400 });
}
const preparedContext = vec.slice(0, 3).map(({ metadata }) => `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`).join("\n\n");
const prompt = `You are an agent that summarizes a page based on the query. Be direct and concise, don't say 'based on the context'.\n\n Context:\n${preparedContext} \nAnswer this question based on the context. Question: ${query}\nAnswer:`
const output = await model.generateContentStream(prompt);
const response = new Response(
new ReadableStream({
async start(controller) {
const converter = new TextEncoder();
for await (const chunk of output.stream) {
const chunkText = await chunk.text();
const encodedChunk = converter.encode("data: " + JSON.stringify({ "response": chunkText }) + "\n\n");
controller.enqueue(encodedChunk);
}
const doneChunk = converter.encode("data: [DONE]");
controller.enqueue(doneChunk);
controller.close();
}
})
);
return response;
}

2
apps/extension/.gitignore vendored
View file

@ -1,4 +1,6 @@
# Logs
*.zip
logs
*.log
npm-debug.log*

3
apps/extension/package.json
View file

@ -7,7 +7,8 @@
"dev": "vite",
"build": "tsc && vite build",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview"
"preview": "vite preview",
"package": "zip -r extension.zip dist/"
},
"dependencies": {
"@radix-ui/react-tooltip": "^1.0.7",

4
apps/extension/src/App.tsx
View file

@ -14,7 +14,7 @@ function App() {
if (loginButton) {
if (jwt) {
fetch('https://anycontext.dhr.wtf/api/me', {
fetch('https://supermemory.dhr.wtf/api/me', {
headers: {
Authorization: `Bearer ${jwt}`,
},
@ -43,7 +43,7 @@ function App() {
<button
onClick={() =>
chrome.tabs.create({
url: 'https://anycontext.dhr.wtf/api/auth/signin',
url: 'https://supermemory.dhr.wtf/api/auth/signin',
})
}
id="login"

18
apps/extension/src/SideBar.tsx
View file

@ -214,9 +214,9 @@ function SideBar({ jwt }: { jwt: string }) {
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
className="lucide lucide-send-horizontal"
>
<path d="m3 3 3 9-3 9 19-9Z" />
@ -241,9 +241,9 @@ function SideBar({ jwt }: { jwt: string }) {
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
className="lucide lucide-panel-right-open"
>
<rect width="18" height="18" x="3" y="3" rx="2" />
@ -282,9 +282,9 @@ function SideBar({ jwt }: { jwt: string }) {
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
className="lucide lucide-file-check-2"
>
<path d="M4 22h14a2 2 0 0 0 2-2V7l-5-5H6a2 2 0 0 0-2 2v4" />

4
apps/extension/src/background.ts
View file

@ -17,7 +17,7 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
console.error("No JWT found");
return;
}
fetch("https://anycontext.dhr.wtf/api/store", {
fetch("https://supermemory.dhr.wtf/api/store", {
method: "POST",
headers: {
"Authorization": `Bearer ${jwt}`,
@ -33,7 +33,7 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
const jwt = request.jwt;
(async () => {
await fetch("https://anycontext.dhr.wtf/api/ask", {
await fetch("https://supermemory.dhr.wtf/api/ask", {
method: "POST",
headers: {
"Authorization": `Bearer ${jwt}`,

7
apps/extension/src/content.tsx
View file

@ -8,7 +8,8 @@ window.addEventListener('message', (event) => {
if (
!(
window.location.hostname === 'localhost' ||
window.location.hostname.endsWith('dhr.wtf')
window.location.hostname === 'anycontext.dhr.wtf' ||
window.location.hostname === 'supermemory.dhr.wtf'
)
) {
console.log(
@ -35,10 +36,10 @@ import SideBar from './SideBar';
// get JWT from local storage
const jwt = chrome.storage.local.get('jwt').then((data) => {
return data.jwt;
}) as Promise<string>
}) as Promise<string>;
jwt.then((jwt) => {
ReactDOM.createRoot(
document.getElementById('anycontext-app-container')!,
).render(<SideBar jwt={jwt} />);
});
});

2
apps/web/db/prepare.sql
View file

@ -30,6 +30,7 @@ CREATE TABLE `storedContent` (
`title` text(255),
`description` text(255),
`url` text NOT NULL,
`space` text(255),
`savedAt` integer NOT NULL,
`baseUrl` text(255),
`image` text(255)
@ -63,5 +64,6 @@ CREATE UNIQUE INDEX `storedContent_url_unique` ON `storedContent` (`url`);--> st
CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint
CREATE INDEX `storedContent_savedAt_idx` ON `storedContent` (`savedAt`);--> statement-breakpoint
CREATE INDEX `storedContent_title_idx` ON `storedContent` (`title`);--> statement-breakpoint
CREATE INDEX `storedContent_space_idx` ON `storedContent` (`space`);--> statement-breakpoint
CREATE INDEX `userStoredContent_idx` ON `userStoredContent` (`userId`,`contentId`);--> statement-breakpoint
CREATE UNIQUE INDEX `unique_user_content` ON `userStoredContent` (`userId`,`contentId`);

3
apps/web/package.json
View file

@ -11,7 +11,8 @@
"preview": "bun pages:build && wrangler pages dev .vercel/output/static",
"deploy": "bun pages:build && wrangler pages deploy .vercel/output/static",
"build-cf-types": "wrangler types --env-interface CloudflareEnv env.d.ts",
"prepare-local-db": "drizzle-kit generate:sqlite --out db/prepare.sql && wrangler d1 execute dev-d1-anycontext --local --file=db/prepare.sql"
"prepare-local-db": "drizzle-kit generate:sqlite --out db/prepare.sql && wrangler d1 execute dev-d1-anycontext --local --file=db/prepare.sql",
"schema-change": "drizzle-kit generate:sqlite --out db/schema-change.sql && wrangler d1 execute dev-d1-anycontext --local --file=db/schema-change.sql"
},
"dependencies": {
"@radix-ui/react-avatar": "^1.0.4",

5765
apps/web/pnpm-lock.yaml generated
View file

File diff suppressed because it is too large Load diff

5
apps/web/src/components/QueryAI.tsx
View file

@ -82,6 +82,11 @@ function QueryAI() {
const response = await fetch(`/api/query?q=${input}`);
if (response.status !== 200) {
setIsAiLoading(false);
return;
}
if (response.body) {
let reader = response.body.getReader();
let decoder = new TextDecoder('utf-8');

147
apps/web/src/components/Sidebar.tsx Normal file
View file

@ -0,0 +1,147 @@
'use client';
import { StoredContent } from '@/server/db/schema';
import {
Plus,
MoreHorizontal,
ArrowUpRight,
Edit3,
Trash2,
} from 'lucide-react';
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from './ui/dropdown-menu';
import { useState, useEffect, useRef } from 'react';
export default function Sidebar() {
const websites: StoredContent[] = [
{
id: 1,
content: '',
title: 'Visual Studio Code',
url: 'https://code.visualstudio.com',
description: '',
image: 'https://code.visualstudio.com/favicon.ico',
baseUrl: 'https://code.visualstudio.com',
savedAt: new Date(),
space: 'Development',
},
{
id: 1,
content: '',
title: "yxshv/vscode: An unofficial remake of vscode's landing page",
url: 'https://github.com/yxshv/vscode',
description: '',
image: 'https://github.com/favicon.ico',
baseUrl: 'https://github.com',
savedAt: new Date(),
space: 'Development',
},
];
return (
<aside className="bg-rgray-3 flex h-screen w-[25%] flex-col items-start justify-between py-5 pb-[50vh] font-light">
<div className="flex items-center justify-center gap-1 px-5 text-xl font-normal">
<img src="/brain.png" alt="logo" className="h-10 w-10" />
SuperMemory
</div>
<div className="flex w-full flex-col items-start justify-center p-2">
<h1 className="mb-1 flex w-full items-center justify-center px-3 font-normal">
Websites
<button className="ml-auto ">
<Plus className="h-4 w-4 min-w-4" />
</button>
</h1>
{websites.map((item) => (
<ListItem key={item.id} item={item} />
))}
</div>
</aside>
);
}
export const ListItem: React.FC<{ item: StoredContent }> = ({ item }) => {
const [isEditing, setIsEditing] = useState(false);
const editInputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
if (isEditing) {
setTimeout(() => {
editInputRef.current?.focus();
}, 500);
}
}, [isEditing]);
return (
<div className="hover:bg-rgray-5 focus-within:bg-rgray-5 flex w-full items-center rounded-full py-1 pl-3 pr-2 transition [&:hover>a>[data-upright-icon]]:block [&:hover>a>img]:hidden [&:hover>button]:opacity-100">
<a
href={item.url}
target="_blank"
onClick={(e) => isEditing && e.preventDefault()}
className="flex w-[90%] items-center gap-2 focus:outline-none"
>
{isEditing ? (
<Edit3 className="h-4 w-4" strokeWidth={1.5} />
) : (
<>
<img
src={item.image ?? '/brain.png'}
alt={item.title ?? 'Untitiled website'}
className="h-4 w-4"
/>
<ArrowUpRight
data-upright-icon
className="hidden h-4 w-4 min-w-4 scale-125"
strokeWidth={1.5}
/>
</>
)}
{isEditing ? (
<input
ref={editInputRef}
autoFocus
className="text-rgray-12 w-full bg-transparent focus:outline-none"
placeholder={item.title ?? 'Untitled website'}
onBlur={(e) => setIsEditing(false)}
onKeyDown={(e) => e.key === 'Escape' && setIsEditing(false)}
/>
) : (
<span className="w-full truncate text-nowrap">
{item.title ?? 'Untitled website'}
</span>
)}
</a>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button className="ml-auto w-4 min-w-4 rounded-[0.15rem] opacity-0 focus:opacity-100 focus:outline-none">
<MoreHorizontal className="h-4 w-4 min-w-4" />
</button>
</DropdownMenuTrigger>
<DropdownMenuContent className="w-5">
<DropdownMenuItem onClick={() => window.open(item.url)}>
<ArrowUpRight
className="mr-2 h-4 w-4 scale-125"
strokeWidth={1.5}
/>
Open
</DropdownMenuItem>
<DropdownMenuItem
onClick={(e) => {
setIsEditing(true);
}}
>
<Edit3 className="mr-2 h-4 w-4 " strokeWidth={1.5} />
Edit
</DropdownMenuItem>
<DropdownMenuItem className="focus:bg-red-100 focus:text-red-400 dark:focus:bg-red-100/10">
<Trash2 className="mr-2 h-4 w-4 " strokeWidth={1.5} />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
);
};

2
apps/web/src/server/db/schema.ts
View file

@ -114,6 +114,7 @@ export const storedContent = createTable(
title: text("title", { length: 255 }),
description: text("description", { length: 255 }),
url: text("url").notNull().unique(),
space: text("space", { length: 255 }),
savedAt: int("savedAt", { mode: "timestamp" }).notNull(),
baseUrl: text("baseUrl", { length: 255 }),
image: text("image", { length: 255 }),
@ -122,6 +123,7 @@ export const storedContent = createTable(
urlIdx: index("storedContent_url_idx").on(sc.url),
savedAtIdx: index("storedContent_savedAt_idx").on(sc.savedAt),
titleInx: index("storedContent_title_idx").on(sc.title),
spaceIdx: index("storedContent_space_idx").on(sc.space),
}),
);

1
package.json
View file

@ -39,6 +39,7 @@
"@cloudflare/ai": "^1.0.52",
"@cloudflare/next-on-pages-next-dev": "^0.0.1",
"@crxjs/vite-plugin": "^1.0.14",
"@google/generative-ai": "^0.3.1",
"@heroicons/react": "^2.1.1",
"@langchain/cloudflare": "^0.0.3",
"@radix-ui/colors": "^3.0.0",