scriptoriumadmin/supermemory
1
0
Fork 0
mirror of https://github.com/supermemoryai/supermemory.git synced 2026-08-28 05:25:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

implemented AI with citations and extension flow, using openai embedder

Browse source
This commit is contained in:
Dhravya 2024-02-28 15:36:39 -07:00
parent 7752cf813e
commit 60012606c8
40 changed files with 466 additions and 351 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
apps/cf-ai-backend/src/OpenAIEmbedder.ts Normal file
View file

@ -0,0 +1,43 @@
import { AiTextEmbeddings } from "@cloudflare/ai/dist/tasks/text-embeddings";
interface OpenAIEmbeddingsParams {
apiKey: string;
modelName: string;
}
export class OpenAIEmbeddings {
private apiKey: string;
private modelName: string;
constructor({ apiKey, modelName }: OpenAIEmbeddingsParams) {
this.apiKey = apiKey;
this.modelName = modelName;
}
async embedDocuments(texts: string[]): Promise<number[][]> {
const responses = await Promise.all(texts.map(text => this.embedQuery(text)));
return responses;
}
async embedQuery(text: string): Promise<number[]> {
const response = await fetch('https://api.openai.com/v1/embeddings', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
},
body: JSON.stringify({
input: text,
model: this.modelName
})
});
const data = await response.json() as {
data: {
embedding: number[]
}[]
};
return data.data[0].embedding;
}
}

28
apps/cf-ai-backend/src/index.ts
View file

@ -6,14 +6,15 @@ import type {
import {
CloudflareVectorizeStore,
CloudflareWorkersAIEmbeddings,
} from "@langchain/cloudflare";
import { Ai } from '@cloudflare/ai';
import { OpenAIEmbeddings } from "./OpenAIEmbedder";
export interface Env {
VECTORIZE_INDEX: VectorizeIndex;
AI: Fetcher;
SECURITY_KEY: string
SECURITY_KEY: string;
OPENAI_API_KEY: string;
}
@ -28,17 +29,18 @@ export default {
}
const pathname = new URL(request.url).pathname;
const embeddings = new CloudflareWorkersAIEmbeddings({
binding: env.AI,
modelName: "@cf/baai/bge-small-en-v1.5",
const embeddings = new OpenAIEmbeddings({
apiKey: env.OPENAI_API_KEY,
modelName: 'text-embedding-3-small',
});
const store = new CloudflareVectorizeStore(embeddings, {
index: env.VECTORIZE_INDEX,
});
const ai = new Ai(env.AI)
if (pathname === "/add" && request.method === "POST") {
const body = await request.json() as {
pageContent: string,
title?: string,
@ -47,14 +49,16 @@ export default {
user: string
};
if (!body.pageContent || !body.url) {
return new Response(JSON.stringify({ message: "Invalid Page Content" }), { status: 400 });
}
const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${body.pageContent}`
await store.addDocuments([
{
pageContent: body.pageContent,
pageContent: newPageContent,
metadata: {
title: body.title ?? "",
description: body.description ?? "",
@ -88,7 +92,7 @@ export default {
const resp = await store.similaritySearch(query, topK, filter)
if (resp.length ===0) {
if (resp.length === 0) {
return new Response(JSON.stringify({ message: "No Results Found" }), { status: 400 });
}
@ -96,7 +100,13 @@ export default {
prompt: `You are an agent that summarizes a page based on the query. Be direct and concise, don't say 'based on the context'.\n\n Context:\n${JSON.stringify(resp)} \nAnswer this question based on the context. Question: ${query}`,
})
return new Response(JSON.stringify(output), { status: 200 });
const cleanCitations = resp.map(
({ metadata }) => ({ url: metadata.url, title: metadata.title, description: metadata.description })
)
return new Response(JSON.stringify({
output, citations: cleanCitations
}), { status: 200 });
}
},
};

3
apps/extension/manifest.json
View file

@ -12,7 +12,8 @@
],
"matches": [
"http://localhost:3000/*",
"https://anycontext.dhr.wtf/*"
"https://anycontext.dhr.wtf/*",
"<all_urls>"
]
}
],

81
apps/extension/src/App.tsx
View file

@ -3,67 +3,68 @@ import { z } from 'zod';
import { userObj } from './types/zods';
function App() {
const [count] = useState(0);
const [userData, setUserData] = useState<z.infer<typeof userObj> | null>(
null,
);
useEffect(() => {
const doStuff = () => {
chrome.runtime.sendMessage({ type: 'getJwt' }, (response) => {
const jwt = response.jwt;
const loginButton = document.getElementById('login');
const doStuff = () => {
chrome.runtime.sendMessage({ type: 'getJwt' }, (response) => {
const jwt = response.jwt;
const loginButton = document.getElementById('login');
if (loginButton) {
console.log('JWT', jwt);
if (jwt) {
console.log('DOING STUFF AND JWT');
fetch('http://localhost:3000/api/me', {
headers: {
Authorization: `Bearer ${jwt}`,
},
})
.then((res) => res.json())
.then((data) => {
console.log(data);
const d = userObj.safeParse(data);
if (d.success) {
setUserData(d.data);
} else {
console.error(d.error);
}
});
loginButton.style.display = 'none';
} else {
loginButton.style.display = 'block';
loginButton.addEventListener('click', () => {
chrome.tabs.create({
url: 'http://localhost:3000/api/auth/signin',
});
if (loginButton) {
console.log('JWT', jwt);
if (jwt) {
console.log('DOING STUFF AND JWT');
fetch('http://localhost:3000/api/me', {
headers: {
Authorization: `Bearer ${jwt}`,
},
})
.then((res) => res.json())
.then((data) => {
console.log(data);
const d = userObj.safeParse(data);
if (d.success) {
setUserData(d.data);
} else {
console.error(d.error);
}
});
}
loginButton.style.display = 'none';
}
});
};
}
});
};
useEffect(() => {
doStuff();
}, [count]);
}, []);
return (
<div className="p-8">
<button id="login">Log in</button>
<button
onClick={() =>
chrome.tabs.create({
url: 'http://localhost:3000/api/auth/signin',
})
}
id="login"
>
Log in
</button>
<div>
{userData && (
<div className="flex items-center">
<img
width={40}
className="rounded-full"
src={userData.data[0].user.image!}
src={userData.data.user.image!}
alt=""
/>
<div>
<h3>{userData.data[0].user.name}</h3>
<p>{userData.data[0].user.email}</p>
<h3>{userData.data.user.name}</h3>
<p>{userData.data.user.email}</p>
</div>
</div>
)}

34
apps/extension/src/background.ts
View file

@ -1,9 +1,31 @@
chrome.runtime.onMessage.addListener((request, _, sendResponse) => {
if (request.type === "getJwt") {
if (request.type === "getJwt") {
chrome.storage.local.get(["jwt"], ({ jwt }) => {
sendResponse({ jwt });
});
return true;
}
else if (request.type === "urlChange") {
const content = request.content;
const url = request.url;
console.log(content);
(async () => {
chrome.storage.local.get(["jwt"], ({ jwt }) => {
sendResponse({ jwt });
if (!jwt) {
console.error("No JWT found");
return;
}
fetch("http://localhost:3000/api/store", {
method: "POST",
headers: {
"Authorization": `Bearer ${jwt}`,
},
body: JSON.stringify({ pageContent: content, url }),
}).then(ers => console.log(ers.status))
});
return true;
}
});
})();
}
});

47
apps/extension/src/content.tsx
View file

@ -1,16 +1,45 @@
window.addEventListener("message", (event) => {
window.addEventListener('message', (event) => {
if (event.source !== window) {
return;
}
const { jwt } = event.data;
if (jwt) {
chrome.storage.local.set({ jwt }, () => {
console.log("JWT saved to local storage", jwt);
});
} else if (jwt === undefined) {
chrome.storage.local.remove("jwt", () => {
console.log("JWT removed from local storage");
if (
!(
window.location.hostname === 'localhost' ||
window.location.hostname === 'anycontext.dhr.wtf'
)
) {
console.log(
'JWT is only allowed to be used on localhost or anycontext.dhr.wtf',
);
return;
}
)
chrome.storage.local.set({ jwt }, () => {
console.log('JWT saved to local storage', jwt);
});
}
});
});
// Run when the URL changes, including hash changes
window.onpopstate = sendUrlToAPI;
// Also run when the page loads
sendUrlToAPI();
function sendUrlToAPI() {
// get the current URL
const url = window.location.href;
const blacklist = ['localhost:3000', 'anycontext.dhr.wtf'];
// check if the URL is blacklisted
if (blacklist.some((blacklisted) => url.includes(blacklisted))) {
console.log('URL is blacklisted');
return;
} else {
// const content = Entire page content, but cleaned up for the LLM. No ads, no scripts, no styles, just the text. if article, just the importnat info abou tit.
const content = document.documentElement.innerText;
chrome.runtime.sendMessage({ type: 'urlChange', content, url });
}
}

4
apps/extension/src/types/zods.ts
View file

@ -2,7 +2,7 @@ import { z } from "zod"
export const userObj = z.object({
message: z.string(),
data: z.array(z.object({
data: z.object({
session: z.object({
sessionToken: z.string(),
userId: z.string(),
@ -15,5 +15,5 @@ export const userObj = z.object({
emailVerified: z.string().nullable(),
image: z.string().nullable().optional()
})
}))
})
})

2
apps/web/.next/BUILD_ID
View file

@ -1 +1 @@
GE_fpFjPpwi13NmQCcZsW
vBsVfgmJuMkLh6-mh3OuR

10
apps/web/.next/app-build-manifest.json
View file

@ -1,22 +1,22 @@
{
"pages": {
"/not-found": [
"static/chunks/webpack-409f1dd28331797e.js",
"static/chunks/webpack-21d4c12a2c3dbbeb.js",
"static/chunks/30b509c0-d7721ce4b2012053.js",
"static/chunks/25-2f3c60275645c813.js",
"static/chunks/main-app-8b951cccf46caf8d.js",
"static/chunks/app/not-found-dbc30055295c6650.js"
],
"/layout": [
"static/chunks/webpack-409f1dd28331797e.js",
"static/chunks/webpack-21d4c12a2c3dbbeb.js",
"static/chunks/30b509c0-d7721ce4b2012053.js",
"static/chunks/25-2f3c60275645c813.js",
"static/chunks/main-app-8b951cccf46caf8d.js",
"static/css/1327fa673efb0c48.css",
"static/chunks/app/layout-3f46cd5460fe4d0d.js"
"static/css/121145655fed7988.css",
"static/chunks/app/layout-50c1233b87ae0cb3.js"
],
"/page": [
"static/chunks/webpack-409f1dd28331797e.js",
"static/chunks/webpack-21d4c12a2c3dbbeb.js",
"static/chunks/30b509c0-d7721ce4b2012053.js",
"static/chunks/25-2f3c60275645c813.js",
"static/chunks/main-app-8b951cccf46caf8d.js",

2
apps/web/.next/app-path-routes-manifest.json
View file

@ -1 +1 @@
{"/favicon.ico/route":"/favicon.ico","/api/[...nextauth]/route":"/api/[...nextauth]","/_not-found":"/_not-found","/page":"/","/api/me/route":"/api/me","/api/hello/route":"/api/hello","/api/store/route":"/api/store","/api/query/route":"/api/query"}
{"/favicon.ico/route":"/favicon.ico","/_not-found":"/_not-found","/api/[...nextauth]/route":"/api/[...nextauth]","/api/me/route":"/api/me","/api/hello/route":"/api/hello","/api/query/route":"/api/query","/page":"/","/api/store/route":"/api/store"}

10
apps/web/.next/build-manifest.json
View file

@ -5,24 +5,24 @@
"devFiles": [],
"ampDevFiles": [],
"lowPriorityFiles": [
"static/GE_fpFjPpwi13NmQCcZsW/_buildManifest.js",
"static/GE_fpFjPpwi13NmQCcZsW/_ssgManifest.js"
"static/vBsVfgmJuMkLh6-mh3OuR/_buildManifest.js",
"static/vBsVfgmJuMkLh6-mh3OuR/_ssgManifest.js"
],
"rootMainFiles": [
"static/chunks/webpack-409f1dd28331797e.js",
"static/chunks/webpack-21d4c12a2c3dbbeb.js",
"static/chunks/30b509c0-d7721ce4b2012053.js",
"static/chunks/25-2f3c60275645c813.js",
"static/chunks/main-app-8b951cccf46caf8d.js"
],
"pages": {
"/_app": [
"static/chunks/webpack-409f1dd28331797e.js",
"static/chunks/webpack-21d4c12a2c3dbbeb.js",
"static/chunks/framework-c25027af42eb8c45.js",
"static/chunks/main-6d41ecb2c0d95e72.js",
"static/chunks/pages/_app-508d387925ef2fa9.js"
],
"/_error": [
"static/chunks/webpack-409f1dd28331797e.js",
"static/chunks/webpack-21d4c12a2c3dbbeb.js",
"static/chunks/framework-c25027af42eb8c45.js",
"static/chunks/main-6d41ecb2c0d95e72.js",
"static/chunks/pages/_error-e16765248192e4ee.js"

BIN
apps/web/.next/cache/webpack/client-production/0.pack vendored
View file

Binary file not shown.

BIN
apps/web/.next/cache/webpack/client-production/index.pack vendored
View file

Binary file not shown.

BIN
apps/web/.next/cache/webpack/server-production/index.pack vendored
View file

Binary file not shown.

2
apps/web/.next/prerender-manifest.js
View file

@ -1 +1 @@
self.__PRERENDER_MANIFEST="{\"version\":4,\"routes\":{\"/favicon.ico\":{\"initialHeaders\":{\"cache-control\":\"public, max-age=0, must-revalidate\",\"content-type\":\"image/x-icon\",\"x-next-cache-tags\":\"_N_T_/layout,_N_T_/favicon.ico/layout,_N_T_/favicon.ico/route,_N_T_/favicon.ico\"},\"experimentalBypassFor\":[{\"type\":\"header\",\"key\":\"Next-Action\"},{\"type\":\"header\",\"key\":\"content-type\",\"value\":\"multipart/form-data\"}],\"initialRevalidateSeconds\":false,\"srcRoute\":\"/favicon.ico\",\"dataRoute\":null}},\"dynamicRoutes\":{},\"notFoundRoutes\":[],\"preview\":{\"previewModeId\":\"c0dbc02cfe143a5c6937aa0cbba65268\",\"previewModeSigningKey\":\"5be9361f72b19862e69e55074a415c43ec8bf7d53d4956736abe3a703a869bd3\",\"previewModeEncryptionKey\":\"e9409a8c842417e2c2b7ec0df05a1a93b5aab3efebf2efd084ad26d4707050d6\"}}"
self.__PRERENDER_MANIFEST="{\"version\":4,\"routes\":{\"/favicon.ico\":{\"initialHeaders\":{\"cache-control\":\"public, max-age=0, must-revalidate\",\"content-type\":\"image/x-icon\",\"x-next-cache-tags\":\"_N_T_/layout,_N_T_/favicon.ico/layout,_N_T_/favicon.ico/route,_N_T_/favicon.ico\"},\"experimentalBypassFor\":[{\"type\":\"header\",\"key\":\"Next-Action\"},{\"type\":\"header\",\"key\":\"content-type\",\"value\":\"multipart/form-data\"}],\"initialRevalidateSeconds\":false,\"srcRoute\":\"/favicon.ico\",\"dataRoute\":null}},\"dynamicRoutes\":{},\"notFoundRoutes\":[],\"preview\":{\"previewModeId\":\"4c61acf865da748c0b32a4861a8239c9\",\"previewModeSigningKey\":\"e7f73b1ba195db3ac691ad50bd7a3c07caca4e8bf9a2155ee3416564f4a3ead4\",\"previewModeEncryptionKey\":\"d0bff3f3653e7f217ef0807055a874ea4d15f209001a72222f78493814056d73\"}}"

2
apps/web/.next/prerender-manifest.json
View file

@ -1 +1 @@
{"version":4,"routes":{"/favicon.ico":{"initialHeaders":{"cache-control":"public, max-age=0, must-revalidate","content-type":"image/x-icon","x-next-cache-tags":"_N_T_/layout,_N_T_/favicon.ico/layout,_N_T_/favicon.ico/route,_N_T_/favicon.ico"},"experimentalBypassFor":[{"type":"header","key":"Next-Action"},{"type":"header","key":"content-type","value":"multipart/form-data"}],"initialRevalidateSeconds":false,"srcRoute":"/favicon.ico","dataRoute":null}},"dynamicRoutes":{},"notFoundRoutes":[],"preview":{"previewModeId":"c0dbc02cfe143a5c6937aa0cbba65268","previewModeSigningKey":"5be9361f72b19862e69e55074a415c43ec8bf7d53d4956736abe3a703a869bd3","previewModeEncryptionKey":"e9409a8c842417e2c2b7ec0df05a1a93b5aab3efebf2efd084ad26d4707050d6"}}
{"version":4,"routes":{"/favicon.ico":{"initialHeaders":{"cache-control":"public, max-age=0, must-revalidate","content-type":"image/x-icon","x-next-cache-tags":"_N_T_/layout,_N_T_/favicon.ico/layout,_N_T_/favicon.ico/route,_N_T_/favicon.ico"},"experimentalBypassFor":[{"type":"header","key":"Next-Action"},{"type":"header","key":"content-type","value":"multipart/form-data"}],"initialRevalidateSeconds":false,"srcRoute":"/favicon.ico","dataRoute":null}},"dynamicRoutes":{},"notFoundRoutes":[],"preview":{"previewModeId":"4c61acf865da748c0b32a4861a8239c9","previewModeSigningKey":"e7f73b1ba195db3ac691ad50bd7a3c07caca4e8bf9a2155ee3416564f4a3ead4","previewModeEncryptionKey":"d0bff3f3653e7f217ef0807055a874ea4d15f209001a72222f78493814056d73"}}

8
apps/web/.next/server/app-paths-manifest.json
View file

@ -1,10 +1,10 @@
{
"/favicon.ico/route": "app/favicon.ico/route.js",
"/api/[...nextauth]/route": "app/api/[...nextauth]/route.js",
"/_not-found": "app/_not-found.js",
"/page": "app/page.js",
"/api/[...nextauth]/route": "app/api/[...nextauth]/route.js",
"/api/me/route": "app/api/me/route.js",
"/api/hello/route": "app/api/hello/route.js",
"/api/store/route": "app/api/store/route.js",
"/api/query/route": "app/api/query/route.js"
"/api/query/route": "app/api/query/route.js",
"/page": "app/page.js",
"/api/store/route": "app/api/store/route.js"
}

10
apps/web/.next/server/app/_not-found.js
View file

File diff suppressed because one or more lines are too long

2
apps/web/.next/server/app/_not-found_client-reference-manifest.js
View file

File diff suppressed because one or more lines are too long

10
apps/web/.next/server/app/api/store/route.js
View file

File diff suppressed because one or more lines are too long

18
apps/web/.next/server/app/page.js
View file

File diff suppressed because one or more lines are too long

2
apps/web/.next/server/app/page_client-reference-manifest.js
View file

File diff suppressed because one or more lines are too long

2
apps/web/.next/server/functions-config-manifest.json
View file

@ -1 +1 @@
{"version":1,"functions":{"/api/[...nextauth]":{},"/_not-found":{},"/api/store":{},"/api/me":{},"/":{},"/api/query":{},"/api/hello":{}}}
{"version":1,"functions":{"/api/[...nextauth]":{},"/_not-found":{},"/api/query":{},"/api/store":{},"/api/hello":{},"/api/me":{},"/":{}}}

2
apps/web/.next/server/middleware-build-manifest.js
View file

@ -1 +1 @@
self.__BUILD_MANIFEST={polyfillFiles:["static/chunks/polyfills-c67a75d1b6f99dc8.js"],devFiles:[],ampDevFiles:[],lowPriorityFiles:["static/GE_fpFjPpwi13NmQCcZsW/_buildManifest.js","static/GE_fpFjPpwi13NmQCcZsW/_ssgManifest.js"],rootMainFiles:["static/chunks/webpack-409f1dd28331797e.js","static/chunks/30b509c0-d7721ce4b2012053.js","static/chunks/25-2f3c60275645c813.js","static/chunks/main-app-8b951cccf46caf8d.js"],pages:{"/_app":["static/chunks/webpack-409f1dd28331797e.js","static/chunks/framework-c25027af42eb8c45.js","static/chunks/main-6d41ecb2c0d95e72.js","static/chunks/pages/_app-508d387925ef2fa9.js"],"/_error":["static/chunks/webpack-409f1dd28331797e.js","static/chunks/framework-c25027af42eb8c45.js","static/chunks/main-6d41ecb2c0d95e72.js","static/chunks/pages/_error-e16765248192e4ee.js"]},ampFirstPages:[]};
self.__BUILD_MANIFEST={polyfillFiles:["static/chunks/polyfills-c67a75d1b6f99dc8.js"],devFiles:[],ampDevFiles:[],lowPriorityFiles:["static/vBsVfgmJuMkLh6-mh3OuR/_buildManifest.js","static/vBsVfgmJuMkLh6-mh3OuR/_ssgManifest.js"],rootMainFiles:["static/chunks/webpack-21d4c12a2c3dbbeb.js","static/chunks/30b509c0-d7721ce4b2012053.js","static/chunks/25-2f3c60275645c813.js","static/chunks/main-app-8b951cccf46caf8d.js"],pages:{"/_app":["static/chunks/webpack-21d4c12a2c3dbbeb.js","static/chunks/framework-c25027af42eb8c45.js","static/chunks/main-6d41ecb2c0d95e72.js","static/chunks/pages/_app-508d387925ef2fa9.js"],"/_error":["static/chunks/webpack-21d4c12a2c3dbbeb.js","static/chunks/framework-c25027af42eb8c45.js","static/chunks/main-6d41ecb2c0d95e72.js","static/chunks/pages/_error-e16765248192e4ee.js"]},ampFirstPages:[]};

96
apps/web/.next/server/middleware-manifest.json
View file

@ -2,26 +2,6 @@
"sortedMiddleware": [],
"middleware": {},
"functions": {
"/api/[...nextauth]/route": {
"files": [
"server/middleware-build-manifest.js",
"server/middleware-react-loadable-manifest.js",
"server/next-font-manifest.js",
"prerender-manifest.js",
"server/edge-runtime-webpack.js",
"server/app/api/[...nextauth]/route.js"
],
"name": "app/api/[...nextauth]/route",
"page": "/api/[...nextauth]/route",
"matchers": [
{
"regexp": "^/api/(?<nextauth>.+?)$",
"originalSource": "/api/[...nextauth]"
}
],
"wasm": [],
"assets": []
},
"/_not-found": {
"files": [
"server/server-reference-manifest.js",
@ -44,23 +24,21 @@
"wasm": [],
"assets": []
},
"/page": {
"/api/[...nextauth]/route": {
"files": [
"server/server-reference-manifest.js",
"server/app/page_client-reference-manifest.js",
"server/middleware-build-manifest.js",
"server/middleware-react-loadable-manifest.js",
"server/next-font-manifest.js",
"prerender-manifest.js",
"server/edge-runtime-webpack.js",
"server/app/page.js"
"server/app/api/[...nextauth]/route.js"
],
"name": "app/page",
"page": "/page",
"name": "app/api/[...nextauth]/route",
"page": "/api/[...nextauth]/route",
"matchers": [
{
"regexp": "^/$",
"originalSource": "/"
"regexp": "^/api/(?<nextauth>.+?)$",
"originalSource": "/api/[...nextauth]"
}
],
"wasm": [],
@ -106,26 +84,6 @@
"wasm": [],
"assets": []
},
"/api/store/route": {
"files": [
"server/middleware-build-manifest.js",
"server/middleware-react-loadable-manifest.js",
"server/next-font-manifest.js",
"prerender-manifest.js",
"server/edge-runtime-webpack.js",
"server/app/api/store/route.js"
],
"name": "app/api/store/route",
"page": "/api/store/route",
"matchers": [
{
"regexp": "^/api/store$",
"originalSource": "/api/store"
}
],
"wasm": [],
"assets": []
},
"/api/query/route": {
"files": [
"server/middleware-build-manifest.js",
@ -145,6 +103,48 @@
],
"wasm": [],
"assets": []
},
"/page": {
"files": [
"server/server-reference-manifest.js",
"server/app/page_client-reference-manifest.js",
"server/middleware-build-manifest.js",
"server/middleware-react-loadable-manifest.js",
"server/next-font-manifest.js",
"prerender-manifest.js",
"server/edge-runtime-webpack.js",
"server/app/page.js"
],
"name": "app/page",
"page": "/page",
"matchers": [
{
"regexp": "^/$",
"originalSource": "/"
}
],
"wasm": [],
"assets": []
},
"/api/store/route": {
"files": [
"server/middleware-build-manifest.js",
"server/middleware-react-loadable-manifest.js",
"server/next-font-manifest.js",
"prerender-manifest.js",
"server/edge-runtime-webpack.js",
"server/app/api/store/route.js"
],
"name": "app/api/store/route",
"page": "/api/store/route",
"matchers": [
{
"regexp": "^/api/store$",
"originalSource": "/api/store"
}
],
"wasm": [],
"assets": []
}
},
"version": 2

2
apps/web/.next/server/pages-manifest.json
View file

@ -1 +1 @@
{"/_app":"pages/_app.js","/_error":"pages/_error.js","/_document":"pages/_document.js"}
{"/_error":"pages/_error.js","/_app":"pages/_app.js","/_document":"pages/_document.js"}

2
apps/web/.next/server/pages/500.html
View file

@ -1 +1 @@
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>500: Internal Server Error</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" crossorigin="" nomodule="" src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="/_next/static/chunks/webpack-409f1dd28331797e.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/framework-c25027af42eb8c45.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/main-6d41ecb2c0d95e72.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/pages/_app-508d387925ef2fa9.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/pages/_error-e16765248192e4ee.js" defer="" crossorigin=""></script><script src="/_next/static/GE_fpFjPpwi13NmQCcZsW/_buildManifest.js" defer="" crossorigin=""></script><script src="/_next/static/GE_fpFjPpwi13NmQCcZsW/_ssgManifest.js" defer="" crossorigin=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">500</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">Internal Server Error<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json" crossorigin="">{"props":{"pageProps":{"statusCode":500}},"page":"/_error","query":{},"buildId":"GE_fpFjPpwi13NmQCcZsW","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>500: Internal Server Error</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" crossorigin="" nomodule="" src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="/_next/static/chunks/webpack-21d4c12a2c3dbbeb.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/framework-c25027af42eb8c45.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/main-6d41ecb2c0d95e72.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/pages/_app-508d387925ef2fa9.js" defer="" crossorigin=""></script><script src="/_next/static/chunks/pages/_error-e16765248192e4ee.js" defer="" crossorigin=""></script><script src="/_next/static/vBsVfgmJuMkLh6-mh3OuR/_buildManifest.js" defer="" crossorigin=""></script><script src="/_next/static/vBsVfgmJuMkLh6-mh3OuR/_ssgManifest.js" defer="" crossorigin=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">500</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">Internal Server Error<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json" crossorigin="">{"props":{"pageProps":{"statusCode":500}},"page":"/_error","query":{},"buildId":"vBsVfgmJuMkLh6-mh3OuR","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

2
apps/web/.next/server/server-reference-manifest.js
View file

@ -1 +1 @@
self.__RSC_SERVER_MANIFEST="{\"node\":{},\"edge\":{},\"encryptionKey\":\"PoRrL5cujaEXxf2b/M/jGou2uriziZUOM2/U5ZgxW/s=\"}"
self.__RSC_SERVER_MANIFEST="{\"node\":{},\"edge\":{},\"encryptionKey\":\"kE3eW3EJRP4svjDSVinFjdnIuDMD3ljOUiXkxWIma6U=\"}"

2
apps/web/.next/server/server-reference-manifest.json
View file

@ -1 +1 @@
{"node":{},"edge":{},"encryptionKey":"PoRrL5cujaEXxf2b/M/jGou2uriziZUOM2/U5ZgxW/s="}
{"node":{},"edge":{},"encryptionKey":"kE3eW3EJRP4svjDSVinFjdnIuDMD3ljOUiXkxWIma6U="}

4
apps/web/.next/trace
View file

File diff suppressed because one or more lines are too long

2
apps/web/package.json
View file

@ -8,7 +8,7 @@
"start": "next start",
"lint": "next lint",
"pages:build": "pnpm next-on-pages",
"dev": "pnpm pages:build && wrangler pages dev .vercel/output/static --port=3000",
"dev": "pnpm pages:build && wrangler pages dev --live-reload .vercel/output/static --port=3000",
"deploy": "pnpm pages:build && wrangler pages deploy .vercel/output/static"
},
"dependencies": {

1
apps/web/src/app/MessagePoster.tsx
View file

@ -5,7 +5,6 @@ import { useEffect } from 'react';
function MessagePoster({ jwt }: { jwt: string }) {
useEffect(() => {
if (typeof window === 'undefined') return;
window.postMessage({ jwt }, '*');
}, [jwt]);

9
apps/web/src/app/api/me/route.ts
View file

@ -10,11 +10,16 @@ export async function GET(req: NextRequest) {
const token = req.cookies.get("next-auth.session-token")?.value ?? req.cookies.get("__Secure-authjs.session-token")?.value ?? req.cookies.get("authjs.session-token")?.value ?? req.headers.get("Authorization")?.replace("Bearer ", "");
const session = await db.select().from(sessions).where(eq(sessions.sessionToken, token!))
.leftJoin(users, eq(sessions.userId, users.id)).limit(1)
if (!session || session.length === 0) {
return new Response(JSON.stringify({ message: "Invalid Key, session not found." }), { status: 404 });
}
const user = await db.select().from(users).where(eq(users.id, session[0].userId)).limit(1)
if (!user || user.length === 0) {
return NextResponse.json({ message: "Invalid Key, session not found." }, { status: 404 });
}
return new Response(JSON.stringify({ message: "OK", data: session }), { status: 200 });
return new Response(JSON.stringify({ message: "OK", data: { session: session[0], user: user[0] } }), { status: 200 });
}

16
apps/web/src/app/api/query/route.ts
View file

@ -9,25 +9,27 @@ export const runtime = "edge";
export async function GET(req: NextRequest) {
const token = req.cookies.get("next-auth.session-token")?.value ?? req.cookies.get("__Secure-authjs.session-token")?.value ?? req.cookies.get("authjs.session-token")?.value ?? req.headers.get("Authorization")?.replace("Bearer ", "");
const session = await db.select().from(sessions).where(eq(sessions.sessionToken, token!))
.leftJoin(users, eq(sessions.userId, users.id)).limit(1)
const sessionData = await db.select().from(sessions).where(eq(sessions.sessionToken, token!))
if (!session || session.length === 0) {
if (!sessionData || sessionData.length === 0) {
return new Response(JSON.stringify({ message: "Invalid Key, session not found." }), { status: 404 });
}
if (!session[0].user) {
return new Response(JSON.stringify({ message: "Invalid Key, session not found." }), { status: 404 });
const user = await db.select().from(users).where(eq(users.id, sessionData[0].userId)).limit(1)
if (!user || user.length === 0) {
return NextResponse.json({ message: "Invalid Key, session not found." }, { status: 404 });
}
const session = {session: sessionData[0], user: user[0]}
const query = new URL(req.url).searchParams.get("q");
if (!query) {
return new Response(JSON.stringify({ message: "Invalid query" }), { status: 400 });
}
console.log(session[0].user)
const resp = await fetch(`https://cf-ai-backend.dhravya.workers.dev/query?q=${query}&user=${session[0].user.email ?? session[0].user.name}`, {
const resp = await fetch(`https://cf-ai-backend.dhravya.workers.dev/query?q=${query}&user=${session.user.email ?? session.user.name}`, {
headers: {
"X-Custom-Auth-Key": env.BACKEND_SECURITY_KEY,
}

78
apps/web/src/app/api/store/route.ts
View file

@ -1,8 +1,9 @@
import { db } from "@/server/db";
import { eq } from "drizzle-orm";
import { sessions, users } from "@/server/db/schema";
import { sessions, storedContent, userStoredContent, users } from "@/server/db/schema";
import { type NextRequest, NextResponse } from "next/server";
import { env } from "@/env";
import { getMetaData } from "@/server/helpers";
export const runtime = "edge";
@ -12,39 +13,78 @@ export async function POST(req: NextRequest) {
console.log(token ? token : 'token not found lol')
console.log(process.env.DATABASE)
const session = await db.select().from(sessions).where(eq(sessions.sessionToken, token!))
.leftJoin(users, eq(sessions.userId, users.id)).limit(1)
const sessionData = await db.select().from(sessions).where(eq(sessions.sessionToken, token!))
if (!session || session.length === 0) {
if (!sessionData || sessionData.length === 0) {
return new Response(JSON.stringify({ message: "Invalid Key, session not found." }), { status: 404 });
}
const user = await db.select().from(users).where(eq(users.id, sessionData[0].userId)).limit(1)
if (!user || user.length === 0) {
return NextResponse.json({ message: "Invalid Key, session not found." }, { status: 404 });
}
if (!session[0].user) {
return NextResponse.json({ message: "Invalid Key, session not found." }, { status: 404 });
}
const session = { session: sessionData[0], user: user[0] }
const data = await req.json() as {
pageContent: string,
title?: string,
description?: string,
url: string,
};
const metadata = await getMetaData(data.url);
const resp = await fetch("https://cf-ai-backend.dhravya.workers.dev/add", {
method: "POST",
headers: {
"X-Custom-Auth-Key": env.BACKEND_SECURITY_KEY,
},
body: JSON.stringify({ ...data, user: session[0].user.email }),
});
const _ = await resp.json();
let id: number | undefined = undefined;
if (resp.status !== 200) {
const storedCont = await db.select().from(storedContent).where(eq(storedContent.url, data.url)).limit(1)
if (storedCont.length > 0) {
id = storedCont[0].id;
} else {
const storedContentId = await db.insert(storedContent).values({
content: data.pageContent,
title: metadata.title,
description: metadata.description,
url: data.url,
baseUrl: metadata.baseUrl,
image: metadata.image,
savedAt: new Date()
})
id = storedContentId.meta.last_row_id;
}
try {
await db.insert(userStoredContent).values({
userId: session.user.id,
contentId: id
});
} catch (e) {
console.log(e);
}
console.log({ ...data, user: session.user.email })
const res = await Promise.race([
fetch("https://cf-ai-backend.dhravya.workers.dev/add", {
method: "POST",
headers: {
"X-Custom-Auth-Key": env.BACKEND_SECURITY_KEY,
},
body: JSON.stringify({ ...data, user: session.user.email }),
}),
new Promise((_, reject) =>
setTimeout(() => reject(new Error('Request timed out')), 40000)
)
]) as Response
const _ = await res.text();
console.log(_)
if (res.status !== 200) {
return NextResponse.json({ message: "Error", error: "Error in CF function" }, { status: 500 });
}
return NextResponse.json({ message: "OK", data: "Success" }, { status: 200 });
}

187
apps/web/src/components/component.tsx
View file

@ -1,14 +1,19 @@
/**
* This code was generated by v0 by Vercel.
* @see https://v0.dev/t/pva6O4OIeZq
*/
import { Input } from "@/components/ui/input"
import { AvatarImage, AvatarFallback, Avatar } from "@/components/ui/avatar"
import { Button } from "@/components/ui/button"
import { Badge } from "@/components/ui/badge"
import { CardContent, CardFooter, Card } from "@/components/ui/card"
import { Input } from '@/components/ui/input';
import { AvatarImage, AvatarFallback, Avatar } from '@/components/ui/avatar';
import { Button } from '@/components/ui/button';
import { Badge } from '@/components/ui/badge';
import { CardContent, CardFooter, Card } from '@/components/ui/card';
import { db } from '@/server/db';
import { storedContent } from '@/server/db/schema';
import { parser } from 'html-metadata-parser';
import { getMetaData } from '@/server/helpers';
export async function Component() {
// const posts = await db.query.storedContent.findMany({
// where: (users, { eq }) => eq(users.id, 1),
// });
export function Component() {
return (
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<header className="flex justify-between items-center py-6">
@ -19,7 +24,10 @@ export function Component() {
<div className="flex items-center space-x-4">
<Input className="w-72" placeholder="Search..." />
<Avatar>
<AvatarImage alt="User avatar" src="/placeholder.svg?height=32&width=32" />
<AvatarImage
alt="User avatar"
src="/placeholder.svg?height=32&width=32"
/>
<AvatarFallback>U</AvatarFallback>
</Avatar>
<Button className="whitespace-nowrap" variant="outline">
@ -33,144 +41,31 @@ export function Component() {
<Badge variant="secondary">Education & Career (1)</Badge>
</nav>
<main className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6">
<Card className="w-full">
<img
alt="Hard drive"
className="w-full h-48 object-cover"
height="200"
src="/placeholder.svg"
style={{
aspectRatio: "300/200",
objectFit: "cover",
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold">I'd like to sell you a hard drive.</h3>
<p className="text-sm text-gray-600">SUBSTACK.COM</p>
<p className="text-sm">
Zenfetch is a proposed tool aimed to help knowledge workers retain and leverage the knowledge.
</p>
</CardContent>
<CardFooter className="flex justify-between">
<Button variant="ghost">Read More</Button>
</CardFooter>
</Card>
<Card className="w-full">
<img
alt="AI Prompting"
className="w-full h-48 object-cover"
height="200"
src="/placeholder.svg"
style={{
aspectRatio: "300/200",
objectFit: "cover",
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold">A guide to prompting AI (for what it is worth)</h3>
<p className="text-sm text-gray-600">ONEUSEFULTHING.ORG</p>
<p className="text-sm">Summary is still generating. Try refreshing the page in a few seconds.</p>
</CardContent>
<CardFooter className="flex justify-between">
<Button variant="ghost">Read More</Button>
</CardFooter>
</Card>
<Card className="w-full">
<img
alt="Unlocking Creativity"
className="w-full h-48 object-cover"
height="200"
src="/placeholder.svg"
style={{
aspectRatio: "300/200",
objectFit: "cover",
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold">Pixel Perfect: How AI Unlocks Creativity</h3>
<p className="text-sm text-gray-600">DIGITALNATIVE.TECH</p>
<p className="text-sm">Summary is still generating. Try refreshing the page in a few seconds.</p>
</CardContent>
<CardFooter className="flex justify-between">
<Button variant="ghost">Read More</Button>
</CardFooter>
</Card>
<Card className="w-full">
<img
alt="Tolerance for Fiction"
className="w-full h-48 object-cover"
height="200"
src="/placeholder.svg"
style={{
aspectRatio: "300/200",
objectFit: "cover",
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold">
Our Declining Tolerance for Fiction & Wild Concepts Likely To Become
</h3>
<p className="text-sm text-gray-600">ARXIV.ORG</p>
<p className="text-sm">Summary is still generating. Try refreshing the page in a few seconds.</p>
</CardContent>
<CardFooter className="flex justify-between">
<Button variant="ghost">Read More</Button>
</CardFooter>
</Card>
<Card className="w-full">
<img
alt="Graph of Thoughts"
className="w-full h-48 object-cover"
height="200"
src="/placeholder.svg"
style={{
aspectRatio: "300/200",
objectFit: "cover",
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold">
Graph of Thoughts: Solving Elaborate Problems with Large Language Models
</h3>
<p className="text-sm text-gray-600">ARXIV.ORG</p>
<p className="text-sm">Summary is still generating. Try refreshing the page in a few seconds.</p>
</CardContent>
<CardFooter className="flex justify-between">
<Button variant="ghost">Read More</Button>
</CardFooter>
</Card>
<Card className="w-full">
<img
alt="Lacking creativity"
className="w-full h-48 object-cover"
height="200"
src="/placeholder.svg"
style={{
aspectRatio: "300/200",
objectFit: "cover",
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold">You're not lacking creativity, you're overwhelmed</h3>
<p className="text-sm text-gray-600">ARXIV.ORG</p>
<p className="text-sm">Summary is still generating. Try refreshing the page in a few seconds.</p>
</CardContent>
<CardFooter className="flex justify-between">
<Button variant="ghost">Read More</Button>
</CardFooter>
</Card>
{/* {metadata.map((post) => (
<Card className="w-full">
<img
alt="Hard drive"
className="w-full h-48 object-cover"
height="200"
src={post.image}
style={{
aspectRatio: '300/200',
objectFit: 'cover',
}}
width="300"
/>
<CardContent>
<h3 className="text-lg font-semibold mt-4">{post.title}</h3>
<p className="text-sm text-gray-600">{post.baseUrl}</p>
<p className="text-sm">{post.description}</p>
</CardContent>
</Card>
))} */}
</main>
</div>
)
);
}
function FlagIcon(props: React.SVGProps<SVGSVGElement>) {
return (
<svg
@ -188,5 +83,5 @@ function FlagIcon(props: React.SVGProps<SVGSVGElement>) {
<path d="M4 15s1-1 4-1 5 2 8 2 4-1 4-1V3s-1 1-4 1-5-2-8-2-4 1-4 1z" />
<line x1="4" x2="4" y1="22" y2="15" />
</svg>
)
);
}

20
apps/web/src/server/auth.ts
View file

@ -10,16 +10,16 @@ export const {
} = NextAuth({
secret: env.NEXTAUTH_SECRET,
trustHost: true,
callbacks: {
session: ({ session, token }) => ({
...session,
user: {
// ...session.user,
id: token.id as string,
token: token,
},
})
},
// callbacks: {
// session: ({ session, token }) => ({
// ...session,
// user: {
// // ...session.user,
// id: token.id as string,
// token: token,
// },
// })
// },
adapter: DrizzleAdapter(db),
providers: [
Google({

38
apps/web/src/server/db/schema.ts
View file

@ -5,8 +5,9 @@ import {
primaryKey,
sqliteTableCreator,
text,
integer,
unique
} from "drizzle-orm/sqlite-core";
import { type AdapterAccount } from "next-auth/adapters";
/**
* This is an example of how to use the multi-project schema feature of Drizzle ORM. Use the same
@ -32,7 +33,7 @@ export const usersRelations = relations(users, ({ many }) => ({
export const accounts = createTable(
"account",
{
id: text("id", { length: 255 }).notNull().primaryKey(),
id: integer("id").notNull().primaryKey({ autoIncrement: true }),
userId: text("userId", { length: 255 }).notNull().references(() => users.id),
type: text("type", { length: 255 }).notNull(),
provider: text("provider", { length: 255 }).notNull(),
@ -55,7 +56,7 @@ export const accounts = createTable(
export const sessions = createTable(
"session",
{
id: text("id", { length: 255 }).notNull().primaryKey(),
id: integer("id").notNull().primaryKey({ autoIncrement: true }),
sessionToken: text("sessionToken", { length: 255 }).notNull(),
userId: text("userId", { length: 255 }).notNull().references(() => users.id),
expires: int("expires", { mode: "timestamp" }).notNull(),
@ -75,4 +76,35 @@ export const verificationTokens = createTable(
(vt) => ({
compoundKey: primaryKey({ columns: [vt.identifier, vt.token] }),
})
);
export const userStoredContent = createTable(
"userStoredContent",
{
userId: text("userId").notNull().references(() => users.id),
contentId: integer("contentId").notNull().references(() => storedContent.id),
},
(usc) => ({
userContentIdx: index("userStoredContent_idx").on(usc.userId, usc.contentId),
uniqueUserContent: unique("unique_user_content").on(usc.userId, usc.contentId),
})
);
export const storedContent = createTable(
"storedContent",
{
id: integer("id").notNull().primaryKey({ autoIncrement: true }),
content: text("content").notNull(),
title: text("title", { length: 255 }),
description: text("description", { length: 255 }),
url: text("url").notNull().unique(),
savedAt: int("savedAt", { mode: "timestamp" }).notNull(),
baseUrl: text("baseUrl", { length: 255 }),
image: text("image", { length: 255 }),
},
(sc) => ({
urlIdx: index("storedContent_url_idx").on(sc.url),
savedAtIdx: index("storedContent_savedAt_idx").on(sc.savedAt),
titleInx: index("storedContent_title_idx").on(sc.title),
})
);

34
apps/web/src/server/helpers.ts Normal file
View file

@ -0,0 +1,34 @@
export async function getMetaData(url: string) {
const response = await fetch(url);
const html = await response.text();
// Extract the base URL
const baseUrl = new URL(url).origin;
// Extract title
const titleMatch = html.match(/<title>(.*?)<\/title>/);
const title = titleMatch ? titleMatch[1] : 'Title not found';
// Extract meta description
const descriptionMatch = html.match(
/<meta name="description" content="(.*?)"\s*\/?>/,
);
const description = descriptionMatch
? descriptionMatch[1]
: 'Description not found';
// Extract Open Graph image
const imageMatch = html.match(
/<meta property="og:image" content="(.*?)"\s*\/?>/,
);
const image = imageMatch ? imageMatch[1] : 'Image not found';
// Prepare the metadata object
const metadata = {
title,
description,
image,
baseUrl,
};
return metadata;
}

2
package.json
View file

@ -45,6 +45,8 @@
"dotenv-cli": "^7.3.0",
"drizzle-orm": "^0.29.4",
"eslint-plugin-next-on-pages": "^1.9.0",
"html-metadata": "^1.7.1",
"html-metadata-parser": "^2.0.4",
"next-auth": "beta"
}
}