From 4daee14a8254b67152285a4a649f2c306c695a24 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 00:12:58 -0500 Subject: [PATCH 01/27] added way to save content and vectorize them. also refactored code and added a bypass to browser rendering --- apps/browser-rendering | 2 +- apps/web/app/(auth)/auth-buttons.tsx | 2 +- apps/web/app/(auth)/signin/page.tsx | 2 +- apps/web/app/(dash)/chat/chatWindow.tsx | 2 +- apps/web/app/(dash)/chat/page.tsx | 2 +- apps/web/app/(dash)/home/page.tsx | 2 +- apps/web/app/(dash)/layout.tsx | 2 +- apps/web/app/(dash)/memories/page.tsx | 121 ++++++---- apps/web/app/(landing)/page.tsx | 2 +- apps/web/app/actions/doers.ts | 220 +++++++++++++++++- apps/web/app/actions/fetchers.ts | 125 +++++++++- apps/web/app/actions/types.ts | 1 + apps/web/app/api/[...nextauth]/route.ts | 2 +- apps/web/app/api/chat/route.ts | 2 +- apps/web/app/api/ensureAuth.ts | 4 +- apps/web/app/api/getCount/route.ts | 8 +- apps/web/app/api/me/route.ts | 4 +- apps/web/app/api/spaces/route.ts | 4 +- apps/web/app/api/store/route.ts | 31 +-- apps/web/app/ref/page.tsx | 8 +- apps/web/cf-env.d.ts | 12 +- apps/web/env.d.ts | 8 - apps/web/{app/helpers => lib}/constants.ts | 6 + .../web/{app/helpers => }/lib/get-metadata.ts | 0 .../helpers => }/lib/get-theme-button.tsx | 0 .../{app/helpers => }/lib/handle-errors.ts | 0 .../web/{app/helpers => }/lib/searchParams.ts | 0 apps/web/{app/helpers => }/server/auth.ts | 0 apps/web/{app/helpers => }/server/db/index.ts | 0 .../web/{app/helpers => }/server/db/schema.ts | 8 +- package.json | 5 +- 31 files changed, 479 insertions(+), 106 deletions(-) rename apps/web/{app/helpers => lib}/constants.ts (86%) rename apps/web/{app/helpers => }/lib/get-metadata.ts (100%) rename apps/web/{app/helpers => }/lib/get-theme-button.tsx (100%) rename apps/web/{app/helpers => }/lib/handle-errors.ts (100%) rename apps/web/{app/helpers => }/lib/searchParams.ts (100%) rename apps/web/{app/helpers => }/server/auth.ts (100%) rename apps/web/{app/helpers => }/server/db/index.ts (100%) rename apps/web/{app/helpers => }/server/db/schema.ts (96%) diff --git a/apps/browser-rendering b/apps/browser-rendering index b37c9623..f1797d84 160000 --- a/apps/browser-rendering +++ b/apps/browser-rendering @@ -1 +1 @@ -Subproject commit b37c962365a36cf342a31a196f4908f4f1343553 +Subproject commit f1797d84ff322d98041c0909e65ef6db8f7aa2cd diff --git a/apps/web/app/(auth)/auth-buttons.tsx b/apps/web/app/(auth)/auth-buttons.tsx index 0e99213e..5b0ad06e 100644 --- a/apps/web/app/(auth)/auth-buttons.tsx +++ b/apps/web/app/(auth)/auth-buttons.tsx @@ -2,7 +2,7 @@ import { Button } from "@repo/ui/shadcn/button"; import React from "react"; -import { signIn } from "../helpers/server/auth"; +import { signIn } from "../../server/auth"; function SignIn() { return ( diff --git a/apps/web/app/(auth)/signin/page.tsx b/apps/web/app/(auth)/signin/page.tsx index ba84a94a..d7bad8da 100644 --- a/apps/web/app/(auth)/signin/page.tsx +++ b/apps/web/app/(auth)/signin/page.tsx @@ -1,7 +1,7 @@ import Image from "next/image"; import Link from "next/link"; import Logo from "@/public/logo.svg"; -import { signIn } from "@/app/helpers/server/auth"; +import { signIn } from "@/server/auth"; import { Google } from "@repo/ui/components/icons"; export const runtime = "edge"; diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index b631c835..d4c76469 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -19,7 +19,7 @@ import remarkMath from "remark-math"; import rehypeKatex from "rehype-katex"; import rehypeHighlight from "rehype-highlight"; import { code, p } from "./markdownRenderHelpers"; -import { codeLanguageSubset } from "@/app/helpers/constants"; +import { codeLanguageSubset } from "@/lib/constants"; function ChatWindow({ q, diff --git a/apps/web/app/(dash)/chat/page.tsx b/apps/web/app/(dash)/chat/page.tsx index fd4de826..73519851 100644 --- a/apps/web/app/(dash)/chat/page.tsx +++ b/apps/web/app/(dash)/chat/page.tsx @@ -1,5 +1,5 @@ import ChatWindow from "./chatWindow"; -import { chatSearchParamsCache } from "../../helpers/lib/searchParams"; +import { chatSearchParamsCache } from "../../../lib/searchParams"; // @ts-expect-error await import("katex/dist/katex.min.css"); diff --git a/apps/web/app/(dash)/home/page.tsx b/apps/web/app/(dash)/home/page.tsx index b4bafb38..55f2928e 100644 --- a/apps/web/app/(dash)/home/page.tsx +++ b/apps/web/app/(dash)/home/page.tsx @@ -2,7 +2,7 @@ import React from "react"; import Menu from "../menu"; import Header from "../header"; import QueryInput from "./queryinput"; -import { homeSearchParamsCache } from "@/app/helpers/lib/searchParams"; +import { homeSearchParamsCache } from "@/lib/searchParams"; import { getSpaces } from "@/app/actions/fetchers"; async function Page({ diff --git a/apps/web/app/(dash)/layout.tsx b/apps/web/app/(dash)/layout.tsx index b879a2f5..3ec8926e 100644 --- a/apps/web/app/(dash)/layout.tsx +++ b/apps/web/app/(dash)/layout.tsx @@ -1,7 +1,7 @@ import Header from "./header"; import Menu from "./menu"; import { redirect } from "next/navigation"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; import { Toaster } from "@repo/ui/shadcn/sonner"; async function Layout({ children }: { children: React.ReactNode }) { diff --git a/apps/web/app/(dash)/memories/page.tsx b/apps/web/app/(dash)/memories/page.tsx index bc2fcd53..ff746d1d 100644 --- a/apps/web/app/(dash)/memories/page.tsx +++ b/apps/web/app/(dash)/memories/page.tsx @@ -1,14 +1,31 @@ "use client"; +import { getAllUserMemoriesAndSpaces } from "@/app/actions/fetchers"; +import { Space } from "@/app/actions/types"; +import { Content } from "@/server/db/schema"; import { NextIcon, SearchIcon, UrlIcon } from "@repo/ui/icons"; import Image from "next/image"; -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; -function page() { - const [filter, setFilter] = useState("All") - const setFilterfn = (i:string) => setFilter(i) +function Page() { + const [filter, setFilter] = useState("All"); + const setFilterfn = (i: string) => setFilter(i); + + const [search, setSearch] = useState(""); + + const [memoriesAndSpaces, setMemoriesAndSpaces] = useState<{ + memories: Content[]; + spaces: Space[]; + }>({ memories: [], spaces: [] }); + + useEffect(() => { + (async () => { + const { success, data } = await getAllUserMemoriesAndSpaces(); + if (!success ?? !data) return; + setMemoriesAndSpaces({ memories: data.memories, spaces: data.spaces }); + })(); + }, []); - const [search, setSearch] = useState("") return (

@@ -16,41 +33,50 @@ function page() {

-
- - Search icon -
- - +
+ + Search icon +
+
Spaces
- - - + {memoriesAndSpaces.spaces.map((space) => ( + + ))}
Pages
- - - + {memoriesAndSpaces.memories.map((memory) => ( + + ))}
); } -function TabComponent({title, description}: {title:string, description:string}){ +function TabComponent({ + title, + description, +}: { + title: string; + description: string; +}) { return (
- {title.slice(0,2).toUpperCase()} + {title.slice(0, 2).toUpperCase()}
@@ -58,37 +84,50 @@ function TabComponent({title, description}: {title:string, description:string}){
{description}
- Search icon + Search icon
- ) + ); } -function LinkComponent({title, url}: {title:string, url:string}){ +function LinkComponent({ title, url }: { title: string; url: string }) { return (
-
-
- Url icon +
+
+ Url icon +
+
+
+
{title}
+
{url}
-
-
{title}
-
{url}
-
-
- ) + ); } -const FilterMethods = ["All", "Spaces", "Pages", "Notes"] -function Filters({setFilter, filter}:{setFilter: (i:string)=> void, filter: string}){ +const FilterMethods = ["All", "Spaces", "Pages", "Notes"]; +function Filters({ + setFilter, + filter, +}: { + setFilter: (i: string) => void; + filter: string; +}) { return (
- {FilterMethods.map((i)=> { - return
setFilter(i)} className={`transition px-6 py-2 rounded-xl ${i === filter ? "bg-[#21303D] text-[#369DFD]" : "text-[#B3BCC5] bg-[#1F2428] hover:bg-[#1f262d] hover:text-[#76a3cc]"}`}>{i}
+ {FilterMethods.map((i) => { + return ( +
setFilter(i)} + className={`transition px-6 py-2 rounded-xl ${i === filter ? "bg-[#21303D] text-[#369DFD]" : "text-[#B3BCC5] bg-[#1F2428] hover:bg-[#1f262d] hover:text-[#76a3cc]"}`} + > + {i} +
+ ); })}
- ) + ); } -export default page; +export default Page; diff --git a/apps/web/app/(landing)/page.tsx b/apps/web/app/(landing)/page.tsx index 09f94d92..5f8b28b4 100644 --- a/apps/web/app/(landing)/page.tsx +++ b/apps/web/app/(landing)/page.tsx @@ -5,7 +5,7 @@ import Cta from "./Cta"; import { Toaster } from "@repo/ui/shadcn/toaster"; import Features from "./Features"; import Footer from "./footer"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; import { redirect } from "next/navigation"; export const runtime = "edge"; diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index c8a1f3b4..798d40fe 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -1,10 +1,15 @@ "use server"; import { revalidatePath } from "next/cache"; -import { db } from "../helpers/server/db"; -import { space } from "../helpers/server/db/schema"; +import { db } from "../../server/db"; +import { contentToSpace, space, storedContent } from "../../server/db/schema"; import { ServerActionReturnType } from "./types"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; +import { Tweet } from "react-tweet/api"; +import { getMetaData } from "@/lib/get-metadata"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { LIMITS } from "@/lib/constants"; +import { z } from "zod"; export const createSpace = async ( input: string | FormData, @@ -41,3 +46,212 @@ export const createSpace = async ( } } }; + +const typeDecider = (content: string) => { + // if the content is a URL, then it's a page. if its a URL with https://x.com/user/status/123, then it's a tweet. else, it's a note. + // do strict checking with regex + if (content.match(/https?:\/\/[\w\.]+\/[\w]+\/[\w]+\/[\d]+/)) { + return "tweet"; + } else if (content.match(/https?:\/\/[\w\.]+/)) { + return "page"; + } else { + return "note"; + } +}; + +export const limit = async (userId: string, type = "page") => { + const count = await db + .select({ + count: sql`count(*)`.mapWith(Number), + }) + .from(storedContent) + .where(and(eq(storedContent.userId, userId), eq(storedContent.type, type))); + + if (count[0]!.count > LIMITS[type as keyof typeof LIMITS]) { + return false; + } + + return true; +}; + +const getTweetData = async (tweetID: string) => { + const url = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetID}&lang=en&features=tfw_timeline_list%3A%3Btfw_follower_count_sunset%3Atrue%3Btfw_tweet_edit_backend%3Aon%3Btfw_refsrc_session%3Aon%3Btfw_fosnr_soft_interventions_enabled%3Aon%3Btfw_show_birdwatch_pivots_enabled%3Aon%3Btfw_show_business_verified_badge%3Aon%3Btfw_duplicate_scribes_to_settings%3Aon%3Btfw_use_profile_image_shape_enabled%3Aon%3Btfw_show_blue_verified_badge%3Aon%3Btfw_legacy_timeline_sunset%3Atrue%3Btfw_show_gov_verified_badge%3Aon%3Btfw_show_business_affiliate_badge%3Aon%3Btfw_tweet_edit_frontend%3Aon&token=4c2mmul6mnh`; + + const resp = await fetch(url, { + headers: { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + Accept: "application/json", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + Connection: "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Cache-Control": "max-age=0", + TE: "Trailers", + }, + }); + console.log(resp.status); + const data = (await resp.json()) as Tweet; + + return data; +}; + +export const createMemory = async (input: { + content: string; + spaces?: string[]; +}): ServerActionReturnType => { + const data = await auth(); + + if (!data || !data.user || !data.user.id) { + return { error: "Not authenticated", success: false }; + } + + const type = typeDecider(input.content); + + let pageContent = input.content; + let metadata: Awaited>; + + if (!(await limit(data.user.id, type))) { + return { + success: false, + data: 0, + error: `You have exceeded the limit of ${LIMITS[type as keyof typeof LIMITS]} ${type}s.`, + }; + } + + if (type === "page") { + const response = await fetch("https://md.dhr.wtf/?url=" + input.content, { + headers: { + Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + }, + }); + pageContent = await response.text(); + metadata = await getMetaData(pageContent); + } else if (type === "tweet") { + const tweet = await getTweetData(input.content.split("/").pop() as string); + pageContent = JSON.stringify(tweet); + metadata = { + baseUrl: input.content, + description: tweet.text, + image: tweet.user.profile_image_url_https, + title: `Tweet by ${tweet.user.name}`, + }; + } else if (type === "note") { + pageContent = input.content; + const noteId = new Date().getTime(); + metadata = { + baseUrl: `https://supermemory.ai/note/${noteId}`, + description: `Note created at ${new Date().toLocaleString()}`, + image: "https://supermemory.ai/logo.png", + title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, + }; + } else { + return { + success: false, + data: 0, + error: "Invalid type", + }; + } + + let storeToSpaces = input.spaces; + + if (!storeToSpaces) { + storeToSpaces = []; + } + + // Insert into database + const insertResponse = await db + .insert(storedContent) + .values({ + content: pageContent, + title: metadata.title, + description: metadata.description, + url: input.content, + baseUrl: metadata.baseUrl, + image: metadata.image, + savedAt: new Date(), + userId: data.user.id, + type, + }) + .returning({ id: storedContent.id }); + + const contentId = insertResponse[0]?.id; + if (!contentId) { + return { + success: false, + data: 0, + error: "Something went wrong while saving the document to the database", + }; + } + + if (storeToSpaces.length > 0) { + // Adding the many-to-many relationship between content and spaces + const spaceData = await db + .select() + .from(space) + .where( + and(inArray(space.name, storeToSpaces), eq(space.user, data.user.id)), + ) + .all(); + + await Promise.all( + spaceData.map(async (space) => { + await db + .insert(contentToSpace) + .values({ contentId: contentId, spaceId: space.id }); + }), + ); + } + + const vectorSaveResponse = await fetch( + `${process.env.BACKEND_BASE_URL}/api/add`, + { + method: "POST", + body: JSON.stringify({ + pageContent, + title: metadata.title, + description: metadata.description, + url: metadata.baseUrl, + // TODO: now, in the vector store, we are only saving the first space. We need to save all spaces. + space: storeToSpaces[0], + user: data.user.id, + }), + }, + ); + + if (!vectorSaveResponse.ok) { + const errorData = await vectorSaveResponse.text(); + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${errorData}`, + }; + } + + try { + const response = await vectorSaveResponse.json(); + + const expectedResponse = z.object({ status: z.literal("ok") }); + + const parsedResponse = expectedResponse.safeParse(response); + + if (!parsedResponse.success) { + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${parsedResponse.error.message}`, + }; + } + + return { + success: true, + data: 1, + }; + } catch (e) { + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${e}`, + }; + } +}; diff --git a/apps/web/app/actions/fetchers.ts b/apps/web/app/actions/fetchers.ts index 9c2527f0..dc71252e 100644 --- a/apps/web/app/actions/fetchers.ts +++ b/apps/web/app/actions/fetchers.ts @@ -1,10 +1,15 @@ "use server"; -import { eq } from "drizzle-orm"; -import { db } from "../helpers/server/db"; -import { users } from "../helpers/server/db/schema"; +import { eq, inArray, not, sql } from "drizzle-orm"; +import { db } from "../../server/db"; +import { + Content, + contentToSpace, + storedContent, + users, +} from "../../server/db/schema"; import { ServerActionReturnType, Space } from "./types"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; export const getSpaces = async (): ServerActionReturnType => { const data = await auth(); @@ -23,3 +28,115 @@ export const getSpaces = async (): ServerActionReturnType => { return { success: true, data: spacesWithoutUser }; }; + +export const getAllMemories = async ( + freeMemoriesOnly: boolean = false, +): ServerActionReturnType => { + const data = await auth(); + + if (!data || !data.user) { + return { error: "Not authenticated", success: false }; + } + + if (!freeMemoriesOnly) { + // Returns all memories, no matter the space. + const memories = await db.query.storedContent.findMany({ + where: eq(users, data.user.id), + }); + + return { success: true, data: memories }; + } + + // This only returns memories that are not a part of any space. + // This is useful for home page where we want to show a list of spaces and memories. + const contentNotInAnySpace = await db + .select() + .from(storedContent) + .where( + not( + eq( + storedContent.id, + db + .select({ contentId: contentToSpace.contentId }) + .from(contentToSpace), + ), + ), + ) + .execute(); + + return { success: true, data: contentNotInAnySpace }; +}; + +export const getAllUserMemoriesAndSpaces = async (): ServerActionReturnType<{ + spaces: Space[]; + memories: Content[]; +}> => { + const data = await auth(); + + if (!data || !data.user) { + return { error: "Not authenticated", success: false }; + } + + const spaces = await db.query.space.findMany({ + where: eq(users, data.user.id), + }); + + const spacesWithoutUser = spaces.map((space) => { + return { ...space, user: undefined }; + }); + + // const contentCountBySpace = await db + // .select({ + // spaceId: contentToSpace.spaceId, + // count: sql`count(*)`.mapWith(Number), + // }) + // .from(contentToSpace) + // .where( + // inArray( + // contentToSpace.spaceId, + // spacesWithoutUser.map((space) => space.id), + // ), + // ) + // .groupBy(contentToSpace.spaceId) + // .execute(); + + // console.log(contentCountBySpace); + + // get a count with space mappings like spaceID: count (number of memories in that space) + const contentCountBySpace = await db + .select({ + spaceId: contentToSpace.spaceId, + count: sql`count(*)`.mapWith(Number), + }) + .from(contentToSpace) + .where( + inArray( + contentToSpace.spaceId, + spacesWithoutUser.map((space) => space.id), + ), + ) + .groupBy(contentToSpace.spaceId) + .execute(); + + console.log(contentCountBySpace); + + const contentNotInAnySpace = await db + .select() + .from(storedContent) + .where( + not( + eq( + storedContent.id, + db + .select({ contentId: contentToSpace.contentId }) + .from(contentToSpace), + ), + ), + ) + .execute(); + + return { + success: true, + data: { spaces: spacesWithoutUser, memories: contentNotInAnySpace }, + }; +}; diff --git a/apps/web/app/actions/types.ts b/apps/web/app/actions/types.ts index fbf669e2..5c5afc5c 100644 --- a/apps/web/app/actions/types.ts +++ b/apps/web/app/actions/types.ts @@ -1,6 +1,7 @@ export type Space = { id: number; name: string; + numberOfMemories?: number; }; export type ServerActionReturnType = Promise<{ diff --git a/apps/web/app/api/[...nextauth]/route.ts b/apps/web/app/api/[...nextauth]/route.ts index 50807ab1..e19cc16e 100644 --- a/apps/web/app/api/[...nextauth]/route.ts +++ b/apps/web/app/api/[...nextauth]/route.ts @@ -1,2 +1,2 @@ -export { GET, POST } from "../../helpers/server/auth"; +export { GET, POST } from "../../../server/auth"; export const runtime = "edge"; diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index aba8784c..541ced34 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -54,7 +54,7 @@ export async function POST(req: NextRequest) { ); const resp = await fetch( - `https://new-cf-ai-backend.dhravya.workers.dev/api/chat?query=${query}&user=${session.user.email}&sourcesOnly=${sourcesOnly}&spaces=${spaces}`, + `${process.env.BACKEND_BASE_URL}/api/chat?query=${query}&user=${session.user.email}&sourcesOnly=${sourcesOnly}&spaces=${spaces}`, { headers: { Authorization: `Bearer ${process.env.BACKEND_SECURITY_KEY}`, diff --git a/apps/web/app/api/ensureAuth.ts b/apps/web/app/api/ensureAuth.ts index a1401a07..d2fbac0b 100644 --- a/apps/web/app/api/ensureAuth.ts +++ b/apps/web/app/api/ensureAuth.ts @@ -1,6 +1,6 @@ import { NextRequest } from "next/server"; -import { db } from "../helpers/server/db"; -import { sessions, users } from "../helpers/server/db/schema"; +import { db } from "../../server/db"; +import { sessions, users } from "../../server/db/schema"; import { eq } from "drizzle-orm"; export async function ensureAuth(req: NextRequest) { diff --git a/apps/web/app/api/getCount/route.ts b/apps/web/app/api/getCount/route.ts index f760c145..7cd2a2d3 100644 --- a/apps/web/app/api/getCount/route.ts +++ b/apps/web/app/api/getCount/route.ts @@ -1,6 +1,6 @@ -import { db } from "@/app/helpers/server/db"; +import { db } from "@/server/db"; import { and, eq, ne, sql } from "drizzle-orm"; -import { sessions, storedContent, users } from "@/app/helpers/server/db/schema"; +import { sessions, storedContent, users } from "@/server/db/schema"; import { type NextRequest, NextResponse } from "next/server"; import { ensureAuth } from "../ensureAuth"; @@ -20,7 +20,7 @@ export async function GET(req: NextRequest) { .from(storedContent) .where( and( - eq(storedContent.user, session.user.id), + eq(storedContent.userId, session.user.id), eq(storedContent.type, "twitter-bookmark"), ), ); @@ -32,7 +32,7 @@ export async function GET(req: NextRequest) { .from(storedContent) .where( and( - eq(storedContent.user, session.user.id), + eq(storedContent.userId, session.user.id), ne(storedContent.type, "twitter-bookmark"), ), ); diff --git a/apps/web/app/api/me/route.ts b/apps/web/app/api/me/route.ts index 20b6aece..621dcbfe 100644 --- a/apps/web/app/api/me/route.ts +++ b/apps/web/app/api/me/route.ts @@ -1,6 +1,6 @@ -import { db } from "@/app/helpers/server/db"; +import { db } from "@/server/db"; import { eq } from "drizzle-orm"; -import { sessions, users } from "@/app/helpers/server/db/schema"; +import { sessions, users } from "@/server/db/schema"; import { type NextRequest, NextResponse } from "next/server"; export const runtime = "edge"; diff --git a/apps/web/app/api/spaces/route.ts b/apps/web/app/api/spaces/route.ts index c46b02fc..cbed547d 100644 --- a/apps/web/app/api/spaces/route.ts +++ b/apps/web/app/api/spaces/route.ts @@ -1,5 +1,5 @@ -import { db } from "@/app/helpers/server/db"; -import { sessions, space, users } from "@/app/helpers/server/db/schema"; +import { db } from "@/server/db"; +import { sessions, space, users } from "@/server/db/schema"; import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { ensureAuth } from "../ensureAuth"; diff --git a/apps/web/app/api/store/route.ts b/apps/web/app/api/store/route.ts index f96f90cf..cb10db24 100644 --- a/apps/web/app/api/store/route.ts +++ b/apps/web/app/api/store/route.ts @@ -1,4 +1,4 @@ -import { db } from "@/app/helpers/server/db"; +import { db } from "@/server/db"; import { and, eq, sql, inArray } from "drizzle-orm"; import { contentToSpace, @@ -6,10 +6,12 @@ import { storedContent, users, space, -} from "@/app/helpers/server/db/schema"; +} from "@/server/db/schema"; import { type NextRequest, NextResponse } from "next/server"; -import { getMetaData } from "@/app/helpers/lib/get-metadata"; +import { getMetaData } from "@/lib/get-metadata"; import { ensureAuth } from "../ensureAuth"; +import { limit } from "@/app/actions/doers"; +import { LIMITS } from "@/lib/constants"; export const runtime = "edge"; @@ -33,22 +35,13 @@ export async function POST(req: NextRequest) { storeToSpaces = []; } - const count = await db - .select({ - count: sql`count(*)`.mapWith(Number), - }) - .from(storedContent) - .where( - and( - eq(storedContent.user, session.user.id), - eq(storedContent.type, "page"), - ), - ); - - if (count[0]!.count > 100) { + if (!(await limit(session.user.id))) { return NextResponse.json( - { message: "Error", error: "Limit exceeded" }, - { status: 499 }, + { + message: "Error: Ratelimit exceeded", + error: `You have exceeded the limit of ${LIMITS["page"]} pages.`, + }, + { status: 429 }, ); } @@ -62,7 +55,7 @@ export async function POST(req: NextRequest) { baseUrl: metadata.baseUrl, image: metadata.image, savedAt: new Date(), - user: session.user.id, + userId: session.user.id, }) .returning({ id: storedContent.id }); diff --git a/apps/web/app/ref/page.tsx b/apps/web/app/ref/page.tsx index 9ace733a..b51a16bb 100644 --- a/apps/web/app/ref/page.tsx +++ b/apps/web/app/ref/page.tsx @@ -1,9 +1,9 @@ import { Button } from "@repo/ui/shadcn/button"; -import { auth, signIn, signOut } from "../helpers/server/auth"; -import { db } from "../helpers/server/db"; +import { auth, signIn, signOut } from "../../server/auth"; +import { db } from "../../server/db"; import { sql } from "drizzle-orm"; -import { users } from "../helpers/server/db/schema"; -import { getThemeToggler } from "../helpers/lib/get-theme-button"; +import { users } from "../../server/db/schema"; +import { getThemeToggler } from "../../lib/get-theme-button"; export const runtime = "edge"; diff --git a/apps/web/cf-env.d.ts b/apps/web/cf-env.d.ts index 98303f35..e98c36cf 100644 --- a/apps/web/cf-env.d.ts +++ b/apps/web/cf-env.d.ts @@ -1,6 +1,16 @@ declare global { namespace NodeJS { - interface ProcessEnv extends CloudflareEnv {} + interface ProcessEnv extends CloudflareEnv { + GOOGLE_CLIENT_ID: string; + GOOGLE_CLIENT_SECRET: string; + AUTH_SECRET: string; + R2_ENDPOINT: string; + R2_ACCESS_ID: string; + R2_SECRET_KEY: string; + R2_BUCKET_NAME: string; + BACKEND_SECURITY_KEY: string; + BACKEND_BASE_URL: string; + } } } diff --git a/apps/web/env.d.ts b/apps/web/env.d.ts index 2755280c..4f11ba55 100644 --- a/apps/web/env.d.ts +++ b/apps/web/env.d.ts @@ -2,14 +2,6 @@ // by running `wrangler types --env-interface CloudflareEnv env.d.ts` interface CloudflareEnv { - GOOGLE_CLIENT_ID: string; - GOOGLE_CLIENT_SECRET: string; - AUTH_SECRET: string; - R2_ENDPOINT: string; - R2_ACCESS_ID: string; - R2_SECRET_KEY: string; - R2_BUCKET_NAME: string; - BACKEND_SECURITY_KEY: string; STORAGE: R2Bucket; DATABASE: D1Database; } diff --git a/apps/web/app/helpers/constants.ts b/apps/web/lib/constants.ts similarity index 86% rename from apps/web/app/helpers/constants.ts rename to apps/web/lib/constants.ts index c3fc640a..7a9485cf 100644 --- a/apps/web/app/helpers/constants.ts +++ b/apps/web/lib/constants.ts @@ -1,3 +1,9 @@ +export const LIMITS = { + page: 100, + tweet: 1000, + note: 1000, +}; + export const codeLanguageSubset = [ "python", "javascript", diff --git a/apps/web/app/helpers/lib/get-metadata.ts b/apps/web/lib/get-metadata.ts similarity index 100% rename from apps/web/app/helpers/lib/get-metadata.ts rename to apps/web/lib/get-metadata.ts diff --git a/apps/web/app/helpers/lib/get-theme-button.tsx b/apps/web/lib/get-theme-button.tsx similarity index 100% rename from apps/web/app/helpers/lib/get-theme-button.tsx rename to apps/web/lib/get-theme-button.tsx diff --git a/apps/web/app/helpers/lib/handle-errors.ts b/apps/web/lib/handle-errors.ts similarity index 100% rename from apps/web/app/helpers/lib/handle-errors.ts rename to apps/web/lib/handle-errors.ts diff --git a/apps/web/app/helpers/lib/searchParams.ts b/apps/web/lib/searchParams.ts similarity index 100% rename from apps/web/app/helpers/lib/searchParams.ts rename to apps/web/lib/searchParams.ts diff --git a/apps/web/app/helpers/server/auth.ts b/apps/web/server/auth.ts similarity index 100% rename from apps/web/app/helpers/server/auth.ts rename to apps/web/server/auth.ts diff --git a/apps/web/app/helpers/server/db/index.ts b/apps/web/server/db/index.ts similarity index 100% rename from apps/web/app/helpers/server/db/index.ts rename to apps/web/server/db/index.ts diff --git a/apps/web/app/helpers/server/db/schema.ts b/apps/web/server/db/schema.ts similarity index 96% rename from apps/web/app/helpers/server/db/schema.ts rename to apps/web/server/db/schema.ts index e3e789c6..1ff23c82 100644 --- a/apps/web/app/helpers/server/db/schema.ts +++ b/apps/web/server/db/schema.ts @@ -103,11 +103,9 @@ export const storedContent = createTable( savedAt: int("savedAt", { mode: "timestamp" }).notNull(), baseUrl: text("baseUrl", { length: 255 }), ogImage: text("ogImage", { length: 255 }), - type: text("type", { enum: ["note", "page", "twitter-bookmark"] }).default( - "page", - ), + type: text("type").default("page"), image: text("image", { length: 255 }), - userId: int("user").references(() => users.id, { + userId: text("user").references(() => users.id, { onDelete: "cascade", }), }, @@ -119,6 +117,8 @@ export const storedContent = createTable( }), ); +export type Content = typeof storedContent.$inferSelect; + export const contentToSpace = createTable( "contentToSpace", { diff --git a/package.json b/package.json index 11e3ba06..dd384010 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "devDependencies": { "@clack/prompts": "^0.7.0", "@cloudflare/next-on-pages": "1", - "@cloudflare/workers-types": "^4.20240512.0", + "@cloudflare/workers-types": "^4.20240614.0", "@repo/eslint-config": "*", "@repo/tailwind-config": "*", "@repo/typescript-config": "*", @@ -45,7 +45,7 @@ "@auth/drizzle-adapter": "^1.1.0", "@aws-sdk/client-s3": "^3.577.0", "@aws-sdk/s3-request-presigner": "^3.577.0", - "@cloudflare/puppeteer": "^0.0.8", + "@cloudflare/puppeteer": "^0.0.11", "@headlessui/react": "^2.0.4", "@hono/swagger-ui": "^0.2.2", "@hookform/resolvers": "^3.4.2", @@ -78,6 +78,7 @@ "react-dropzone": "^14.2.3", "react-hook-form": "^7.51.5", "react-markdown": "^9.0.1", + "react-tweet": "^3.2.1", "rehype-highlight": "^7.0.0", "rehype-katex": "^7.0.0", "remark-gfm": "^4.0.0", From e0b83adafc76e1f2bf1753f285b0f0919dc6f4e7 Mon Sep 17 00:00:00 2001 From: codetorso Date: Sun, 16 Jun 2024 00:59:56 -0600 Subject: [PATCH 02/27] Update Setup Guide --- SETUP-GUIDE.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/SETUP-GUIDE.md b/SETUP-GUIDE.md index 7d69b545..97c0be45 100644 --- a/SETUP-GUIDE.md +++ b/SETUP-GUIDE.md @@ -13,8 +13,8 @@ 3. Create a `.dev.vars` file in `apps/web` with the following content: ```bash -GOOGLE_CLIENT_ID="-" -GOOGLE_CLIENT_SECRET="-" +GOOGLE_CLIENT_ID="-" // required, visit https://developers.google.com/identity/protocols/oauth2 +GOOGLE_CLIENT_SECRET="-" // required NEXTAUTH_SECRET='nextauthsecret' DATABASE_URL='database.sqlite' NEXTAUTH_URL='http://localhost:3000' @@ -28,10 +28,10 @@ First, edit the `wrangler.toml` file in `apps/web` to point the d1 database to y You can create a d1 database by running this command ``` -wrangler d1 create DATABASE_NAME +bunx wrangler d1 create ``` -And then replace these values +And then replace database_name and database_id with the values ``` [[d1_databases]] @@ -43,10 +43,12 @@ database_id = "YOUR_DB_ID" Simply run this command in `apps/web` ``` -wrangler d1 execute dev-d1-anycontext --local --file=db/prepare.sql +bunx wrangler d1 migrations apply ``` -If it runs, you can set up the cloud database as well by removing the `--local` flag. +If it runs, you can set up the cloud database as well by removing the `--local` flag, + +if you just want to contribute to frontend then just run `bun run dev` in the root of the project and done! (you won't be able to try ai stuff), otherwise continue... 5. You need to host your own worker for the `apps/cf-ai-backend` module. From 375609d4da29e597e16ba35628959dfe964ab9a0 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 11:37:37 -0500 Subject: [PATCH 03/27] form to add content [PENDING LOADING STATE] --- apps/browser-rendering | 2 +- apps/web/app/(dash)/dynamicisland.tsx | 51 ++++++++++++++++----- apps/web/app/actions/doers.ts | 65 ++++++++++++++++----------- 3 files changed, 80 insertions(+), 38 deletions(-) diff --git a/apps/browser-rendering b/apps/browser-rendering index f1797d84..4d21045a 160000 --- a/apps/browser-rendering +++ b/apps/browser-rendering @@ -1 +1 @@ -Subproject commit f1797d84ff322d98041c0909e65ef6db8f7aa2cd +Subproject commit 4d21045a45fdbf56b7483d3704ae0474ebf044fb diff --git a/apps/web/app/(dash)/dynamicisland.tsx b/apps/web/app/(dash)/dynamicisland.tsx index 31f76fda..6fa56fae 100644 --- a/apps/web/app/(dash)/dynamicisland.tsx +++ b/apps/web/app/(dash)/dynamicisland.tsx @@ -4,12 +4,12 @@ import { AddIcon } from "@repo/ui/icons"; import Image from "next/image"; import { AnimatePresence, useMotionValueEvent, useScroll } from "framer-motion"; -import { useEffect, useRef, useState } from "react"; +import { useActionState, useEffect, useRef, useState } from "react"; import { motion } from "framer-motion"; import { Label } from "@repo/ui/shadcn/label"; import { Input } from "@repo/ui/shadcn/input"; import { Textarea } from "@repo/ui/shadcn/textarea"; -import { createSpace } from "../actions/doers"; +import { createMemory, createSpace } from "../actions/doers"; import { Select, SelectContent, @@ -20,6 +20,7 @@ import { import { Space } from "../actions/types"; import { getSpaces } from "../actions/fetchers"; import { toast } from "sonner"; +import { useFormStatus } from "react-dom"; export function DynamicIsland() { const { scrollYProgress } = useScroll(); @@ -253,13 +254,39 @@ function PageForm({ cancelfn: () => void; spaces: Space[]; }) { + const [loading, setLoading] = useState(false); + + const { pending } = useFormStatus(); return ( -
+
{ + const content = e.get("content")?.toString(); + const space = e.get("space")?.toString(); + if (!content) { + toast.error("Content is required"); + return; + } + setLoading(true); + const cont = await createMemory({ + content: content, + spaces: space ? [space] : undefined, + }); + + console.log(cont); + setLoading(false); + if (cont.success) { + toast.success("Memory created"); + } else { + toast.error("Memory creation failed"); + } + }} + className="bg-secondary border border-muted-foreground px-4 py-3 rounded-2xl mt-2 flex flex-col gap-3" + >
- @@ -272,24 +299,28 @@ function PageForm({
+
+ {loading ?
Loading...
: "not loading"} +
-
- cancel -
+ Submit +
-
+ ); } diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index 798d40fe..3aac1f5a 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -126,7 +126,7 @@ export const createMemory = async (input: { }, }); pageContent = await response.text(); - metadata = await getMetaData(pageContent); + metadata = await getMetaData(input.content); } else if (type === "tweet") { const tweet = await getTweetData(input.content.split("/").pop() as string); pageContent = JSON.stringify(tweet); @@ -159,6 +159,36 @@ export const createMemory = async (input: { storeToSpaces = []; } + const vectorSaveResponse = await fetch( + `${process.env.BACKEND_BASE_URL}/api/add`, + { + method: "POST", + body: JSON.stringify({ + pageContent, + title: metadata.title, + description: metadata.description, + url: metadata.baseUrl, + // TODO: now, in the vector store, we are only saving the first space. We need to save all spaces. + space: storeToSpaces[0], + user: data.user.id, + }), + headers: { + "Content-Type": "application/json", + Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + }, + }, + ); + + if (!vectorSaveResponse.ok) { + const errorData = await vectorSaveResponse.text(); + console.log(errorData); + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${errorData}`, + }; + } + // Insert into database const insertResponse = await db .insert(storedContent) @@ -190,7 +220,13 @@ export const createMemory = async (input: { .select() .from(space) .where( - and(inArray(space.name, storeToSpaces), eq(space.user, data.user.id)), + and( + inArray( + space.id, + storeToSpaces.map((s) => parseInt(s)), + ), + eq(space.user, data.user.id), + ), ) .all(); @@ -203,31 +239,6 @@ export const createMemory = async (input: { ); } - const vectorSaveResponse = await fetch( - `${process.env.BACKEND_BASE_URL}/api/add`, - { - method: "POST", - body: JSON.stringify({ - pageContent, - title: metadata.title, - description: metadata.description, - url: metadata.baseUrl, - // TODO: now, in the vector store, we are only saving the first space. We need to save all spaces. - space: storeToSpaces[0], - user: data.user.id, - }), - }, - ); - - if (!vectorSaveResponse.ok) { - const errorData = await vectorSaveResponse.text(); - return { - success: false, - data: 0, - error: `Failed to save to vector store. Backend returned error: ${errorData}`, - }; - } - try { const response = await vectorSaveResponse.json(); From 5cb5bcdbda329b6935291113a2034943da9a635a Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 11:58:27 -0500 Subject: [PATCH 04/27] fixed a bug --- apps/cf-ai-backend/src/index.ts | 18 +++++------------- apps/cf-ai-backend/src/prompts/prompt1.ts | 3 +-- apps/web/app/(dash)/chat/chatWindow.tsx | 12 ++++++++++++ apps/web/app/api/chat/route.ts | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 2dbb2d0c..36dc6750 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -101,7 +101,7 @@ app.post( const body = c.req.valid("json"); const sourcesOnly = query.sourcesOnly === "true"; - const spaces = query.spaces?.split(",") ?? [""]; + const spaces = query.spaces?.split(",") ?? [undefined]; // Get the AI model maker and vector store const { model, store } = await initQuery(c, query.model); @@ -118,7 +118,7 @@ app.post( // SLICED to 5 to avoid too many queries for (const space of spaces.slice(0, 5)) { console.log("space", space); - if (space !== "") { + if (!space && spaces.length > 1) { // it's possible for space list to be [undefined] so we only add space filter conditionally filter.space = space; } @@ -183,19 +183,11 @@ app.post( return c.json({ ids: storedContent }); } - const vec = responses.matches.map((data) => ({ metadata: data.metadata })); - - const vecWithScores = vec.map((v, i) => ({ - ...v, - score: sortedHighScoreData[i].score, - normalisedScore: sortedHighScoreData[i].normalizedScore, - })); - - const preparedContext = vecWithScores.map( - ({ metadata, score, normalisedScore }) => ({ + const preparedContext = normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, score, - normalisedScore, + normalizedScore, }), ); diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts index d2ee988c..289495b6 100644 --- a/apps/cf-ai-backend/src/prompts/prompt1.ts +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -18,13 +18,12 @@ export const template = ({ contexts, question }) => { // Map over contexts to generate the context and score parts const contextParts = contexts .map( - ({ context, score, normalisedScore }) => ` + ({ context, normalisedScore }) => ` ${context} - score: ${score} normalisedScore: ${normalisedScore} `, ) diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index d4c76469..2473de04 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -46,6 +46,18 @@ function ChatWindow({ const router = useRouter(); const getAnswer = async (query: string, spaces: string[]) => { + const sourcesFetch = await fetch( + `/api/chat?q=${query}&spaces=${spaces}&sourcesOnly=true`, + { + method: "POST", + body: JSON.stringify({ chatHistory }), + }, + ); + + // TODO: handle this properly + const sources = await sourcesFetch.json(); + console.log(sources); + const resp = await fetch(`/api/chat?q=${query}&spaces=${spaces}`, { method: "POST", body: JSON.stringify({ chatHistory }), diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index 541ced34..c19ce92b 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -54,7 +54,7 @@ export async function POST(req: NextRequest) { ); const resp = await fetch( - `${process.env.BACKEND_BASE_URL}/api/chat?query=${query}&user=${session.user.email}&sourcesOnly=${sourcesOnly}&spaces=${spaces}`, + `${process.env.BACKEND_BASE_URL}/api/chat?query=${query}&user=${session.user.id}&sourcesOnly=${sourcesOnly}&spaces=${spaces}`, { headers: { Authorization: `Bearer ${process.env.BACKEND_SECURITY_KEY}`, From 2ffb0542ebff501f4286695dc2f5f494cceaee34 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 12:06:37 -0500 Subject: [PATCH 05/27] proper URLs --- apps/cf-ai-backend/src/helper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 87495c59..e0638ca6 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -98,7 +98,7 @@ export async function batchCreateChunksAndEmbeddings({ chunks: string[]; context: Context<{ Bindings: Env }>; }) { - const ourID = `${body.url}-${body.user}`; + const ourID = `${body.url}/#supermemory-${body.user}`; await deleteDocument({ url: body.url, user: body.user, c: context, store }); From a9f5ebc9e41eb2963d4307d6fc21878c71f340f9 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 12:25:36 -0500 Subject: [PATCH 06/27] include metadata in response, add type to metadata --- apps/cf-ai-backend/src/helper.ts | 1 + apps/cf-ai-backend/src/index.ts | 4 +++- apps/cf-ai-backend/src/types.ts | 1 + apps/web/app/(dash)/chat/chatWindow.tsx | 20 +++++++++++++++++++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index e0638ca6..5d01c881 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -124,6 +124,7 @@ export async function batchCreateChunksAndEmbeddings({ space: body.space ?? "", url: body.url, user: body.user, + type: body.type ?? "page", }, }, ], diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 36dc6750..7d3f69e6 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -180,7 +180,9 @@ app.post( idsAsStrings.map(async (id) => await c.env.KV.get(id)), ); - return c.json({ ids: storedContent }); + const metadata = normalizedData.map((datapoint) => datapoint.metadata); + + return c.json({ ids: storedContent, metadata }); } const preparedContext = normalizedData.map( diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index bea4bf80..ca68d9ee 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -46,4 +46,5 @@ export const vectorObj = z.object({ space: z.string().optional(), url: z.string(), user: z.string(), + type: z.string().optional(), }); diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 2473de04..8956ba9b 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -20,6 +20,8 @@ import rehypeKatex from "rehype-katex"; import rehypeHighlight from "rehype-highlight"; import { code, p } from "./markdownRenderHelpers"; import { codeLanguageSubset } from "@/lib/constants"; +import { z } from "zod"; +import { toast } from "sonner"; function ChatWindow({ q, @@ -56,7 +58,23 @@ function ChatWindow({ // TODO: handle this properly const sources = await sourcesFetch.json(); - console.log(sources); + + const sourcesZod = z.object({ + ids: z.array(z.string()), + metadata: z.array(z.any()), + }); + + const sourcesParsed = sourcesZod.safeParse(sources); + + if (!sourcesParsed.success) { + console.log(sources); + console.error(sourcesParsed.error); + toast.error("Something went wrong while getting the sources"); + return; + } + + console.log(sourcesParsed.data.ids); + console.log(sourcesParsed.data.metadata); const resp = await fetch(`/api/chat?q=${query}&spaces=${spaces}`, { method: "POST", From c5b31e54355909798a163cbbbcdbeedddb2995ee Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 12:29:10 -0500 Subject: [PATCH 07/27] use a dev vectorize database to prevent pollution --- SETUP-GUIDE.md | 3 ++- apps/cf-ai-backend/wrangler.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/SETUP-GUIDE.md b/SETUP-GUIDE.md index 97c0be45..f51b1cb1 100644 --- a/SETUP-GUIDE.md +++ b/SETUP-GUIDE.md @@ -19,6 +19,7 @@ NEXTAUTH_SECRET='nextauthsecret' DATABASE_URL='database.sqlite' NEXTAUTH_URL='http://localhost:3000' BACKEND_SECURITY_KEY='veryrandomsecuritykey' +BACKEND_BASE_URL="where your backend is hosted" ``` 4. Setup the database: @@ -46,7 +47,7 @@ Simply run this command in `apps/web` bunx wrangler d1 migrations apply ``` -If it runs, you can set up the cloud database as well by removing the `--local` flag, +If it runs, you can set up the cloud database as well by removing the `--local` flag, if you just want to contribute to frontend then just run `bun run dev` in the root of the project and done! (you won't be able to try ai stuff), otherwise continue... diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index db0ae945..fa883195 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -5,7 +5,7 @@ node_compat = true [[vectorize]] binding = "VECTORIZE_INDEX" -index_name = "supermem-vector" +index_name = "supermem-vector-dev" [ai] binding = "AI" From 9588768b70be92f9e354a350d14993540405ec61 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 18:17:54 -0500 Subject: [PATCH 08/27] added image support in the backend --- apps/cf-ai-backend/src/helper.ts | 11 ++--- apps/cf-ai-backend/src/index.ts | 83 +++++++++++++++++++++++++++++++- apps/cf-ai-backend/src/types.ts | 2 +- apps/cf-ai-backend/tsconfig.json | 3 +- 4 files changed, 90 insertions(+), 9 deletions(-) diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 5d01c881..1540c9fd 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -103,14 +103,13 @@ export async function batchCreateChunksAndEmbeddings({ await deleteDocument({ url: body.url, user: body.user, c: context, store }); const random = seededRandom(ourID); + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - const uuid = - random().toString(36).substring(2, 15) + - random().toString(36).substring(2, 15) + - "-" + - i; + const chunkId = `${uuid}-${i}`; const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; @@ -129,7 +128,7 @@ export async function batchCreateChunksAndEmbeddings({ }, ], { - ids: [uuid], + ids: [chunkId], }, ); diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 7d3f69e6..94351b70 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -18,7 +18,12 @@ import { swaggerUI } from "@hono/swagger-ui"; const app = new Hono<{ Bindings: Env }>(); -app.get("/doc", swaggerUI({ url: "/doc" })); +app.get( + "/ui", + swaggerUI({ + url: "/doc", + }), +); // ------- MIDDLEWARES ------- app.use("*", poweredBy()); @@ -34,6 +39,17 @@ app.use("/api/", async (c, next) => { }); // ------- MIDDLEWARES END ------- +const fileSchema = z + .instanceof(File) + .refine( + (file) => file.size <= 10 * 1024 * 1024, + "File size should be less than 10MB", + ) // Validate file size + .refine( + (file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type), + "Invalid file type", + ); // Validate file type + app.get("/", (c) => { return c.text("Supermemory backend API is running!"); }); @@ -57,6 +73,71 @@ app.post("/api/add", zValidator("json", vectorObj), async (c) => { return c.json({ status: "ok" }); }); +app.post( + "/api/add-with-image", + zValidator( + "form", + z.object({ + images: z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + "images[]": z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + text: z.string().optional(), + space: z.string().optional(), + url: z.string(), + user: z.string(), + }), + (c) => { + console.log(c); + }, + ), + async (c) => { + const body = c.req.valid("form"); + + const { store } = await initQuery(c); + + if (!(body.images || body["images[]"])) { + return c.json({ status: "error", message: "No images found" }, 400); + } + + const imagePromises = (body.images ?? body["images[]"]).map( + async (image) => { + const buffer = await image.arrayBuffer(); + const input = { + image: [...new Uint8Array(buffer)], + prompt: + "What's in this image? caption everything you see in great detail", + max_tokens: 1024, + }; + const response = await c.env.AI.run( + "@cf/llava-hf/llava-1.5-7b-hf", + input, + ); + console.log(response.description); + return response.description; + }, + ); + + const imageDescriptions = await Promise.all(imagePromises); + + await batchCreateChunksAndEmbeddings({ + store, + body, + chunks: [ + imageDescriptions, + ...(body.text ? chunkText(body.text, 1536) : []), + ].flat(), + context: c, + }); + + return c.json({ status: "ok" }); + }, +); + app.get( "/api/ask", zValidator( diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index ca68d9ee..3db62553 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -2,7 +2,7 @@ import { z } from "zod"; export type Env = { VECTORIZE_INDEX: VectorizeIndex; - AI: Fetcher; + AI: Ai; SECURITY_KEY: string; OPENAI_API_KEY: string; GOOGLE_AI_API_KEY: string; diff --git a/apps/cf-ai-backend/tsconfig.json b/apps/cf-ai-backend/tsconfig.json index 2b75d5a0..fcdf6914 100644 --- a/apps/cf-ai-backend/tsconfig.json +++ b/apps/cf-ai-backend/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "lib": ["ES2020"], - "types": ["@cloudflare/workers-types"] + "types": ["@cloudflare/workers-types"], + "downlevelIteration": true } } From a87030b88b4fb1d9e30f01c8b809c0f69d728e4c Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 18:24:27 -0500 Subject: [PATCH 09/27] OCR try --- apps/cf-ai-backend/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 94351b70..69edbbe7 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -110,7 +110,7 @@ app.post( const input = { image: [...new Uint8Array(buffer)], prompt: - "What's in this image? caption everything you see in great detail", + "What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.", max_tokens: 1024, }; const response = await c.env.AI.run( From 0f3d5c4fc64a7e917dcd0308e648f28988b57fd1 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 18:28:11 -0500 Subject: [PATCH 10/27] store full info --- apps/cf-ai-backend/src/index.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 69edbbe7..75a3b8e8 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -126,7 +126,18 @@ app.post( await batchCreateChunksAndEmbeddings({ store, - body, + body: { + url: body.url, + user: body.user, + type: "image", + description: + imageDescriptions.length > 1 + ? `A group of ${imageDescriptions.length} images on ${body.url}` + : imageDescriptions[0], + space: body.space, + pageContent: imageDescriptions.join("\n"), + title: "Image content from the web", + }, chunks: [ imageDescriptions, ...(body.text ? chunkText(body.text, 1536) : []), From a4c835e58370641947a757cbcb5d3ff4d4c09c06 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 19:00:24 -0500 Subject: [PATCH 11/27] added sources to the response --- apps/cf-ai-backend/src/helper.ts | 1 + apps/web/app/(dash)/chat/chatWindow.tsx | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 1540c9fd..32928250 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -124,6 +124,7 @@ export async function batchCreateChunksAndEmbeddings({ url: body.url, user: body.user, type: body.type ?? "page", + content: newPageContent, }, }, ], diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 8956ba9b..3fd08657 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -73,8 +73,18 @@ function ChatWindow({ return; } - console.log(sourcesParsed.data.ids); - console.log(sourcesParsed.data.metadata); + setChatHistory((prevChatHistory) => { + const newChatHistory = [...prevChatHistory]; + const lastAnswer = newChatHistory[newChatHistory.length - 1]; + if (!lastAnswer) return prevChatHistory; + lastAnswer.answer.sources = sourcesParsed.data.metadata.map((source) => ({ + title: source.title ?? "Untitled", + type: source.type ?? "page", + source: source.url ?? "https://supermemory.ai", + content: source.content ?? "No content available", + })); + return newChatHistory; + }); const resp = await fetch(`/api/chat?q=${query}&spaces=${spaces}`, { method: "POST", From 76c48ccb600d172f362d7cd237094d6082454825 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 19:13:32 -0500 Subject: [PATCH 12/27] add number of chunks to the respnose and only show unique values --- apps/cf-ai-backend/src/types.ts | 2 +- apps/web/app/(dash)/chat/chatWindow.tsx | 37 +++++++++++++++++++++---- apps/web/app/actions/doers.ts | 1 + packages/shared-types/index.ts | 1 + 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index 3db62553..5f6d0583 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -46,5 +46,5 @@ export const vectorObj = z.object({ space: z.string().optional(), url: z.string(), user: z.string(), - type: z.string().optional(), + type: z.string().optional().default("page"), }); diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 3fd08657..77c1f32b 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -22,6 +22,8 @@ import { code, p } from "./markdownRenderHelpers"; import { codeLanguageSubset } from "@/lib/constants"; import { z } from "zod"; import { toast } from "sonner"; +import Link from "next/link"; +import { sources } from "next/dist/compiled/webpack/webpack"; function ChatWindow({ q, @@ -77,11 +79,24 @@ function ChatWindow({ const newChatHistory = [...prevChatHistory]; const lastAnswer = newChatHistory[newChatHistory.length - 1]; if (!lastAnswer) return prevChatHistory; - lastAnswer.answer.sources = sourcesParsed.data.metadata.map((source) => ({ + const filteredSourceUrls = new Set( + sourcesParsed.data.metadata.map((source) => source.url), + ); + const uniqueSources = sourcesParsed.data.metadata.filter((source) => { + if (filteredSourceUrls.has(source.url)) { + filteredSourceUrls.delete(source.url); + return true; + } + return false; + }); + lastAnswer.answer.sources = uniqueSources.map((source) => ({ title: source.title ?? "Untitled", type: source.type ?? "page", source: source.url ?? "https://supermemory.ai", content: source.content ?? "No content available", + numChunks: sourcesParsed.data.metadata.filter( + (f) => f.url === source.url, + ).length, })); return newChatHistory; }); @@ -191,15 +206,25 @@ function ChatWindow({ ))} {chat.answer.sources.map((source, idx) => ( -
-
- {source.type} +
+ {source.type} + + {source.numChunks > 1 && ( + {source.numChunks} chunks + )}
-
{source.title}
-
+
{source.title}
+
+ {source.content.length > 100 + ? source.content.slice(0, 100) + "..." + : source.content} +
+ ))} diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index 3aac1f5a..f94ed8ec 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -171,6 +171,7 @@ export const createMemory = async (input: { // TODO: now, in the vector store, we are only saving the first space. We need to save all spaces. space: storeToSpaces[0], user: data.user.id, + type, }), headers: { "Content-Type": "application/json", diff --git a/packages/shared-types/index.ts b/packages/shared-types/index.ts index 46e3edba..b8792369 100644 --- a/packages/shared-types/index.ts +++ b/packages/shared-types/index.ts @@ -10,6 +10,7 @@ export const ChatHistoryZod = z.object({ source: z.string(), title: z.string(), content: z.string(), + numChunks: z.number().optional().default(1), }), ), }), From be7165529c2486b09783122509b8a837c9889c00 Mon Sep 17 00:00:00 2001 From: codetorso Date: Mon, 17 Jun 2024 03:31:28 -0600 Subject: [PATCH 13/27] gpt-4o is already a default model, so no error-checking required --- apps/cf-ai-backend/src/helper.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 32928250..78ff86da 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -21,8 +21,6 @@ export async function initQuery( index: c.env.VECTORIZE_INDEX, }); - const DEFAULT_MODEL = "gpt-4o"; - let selectedModel: | ReturnType> | ReturnType> @@ -51,13 +49,7 @@ export async function initQuery( selectedModel = openai.chat("gpt-4o"); break; } - - if (!selectedModel) { - throw new Error( - `Model ${model} not found and default model ${DEFAULT_MODEL} is also not available.`, - ); - } - + return { store, model: selectedModel }; } From 88289d12285d98caaa89be424034d91d9ad7f0e2 Mon Sep 17 00:00:00 2001 From: codetorso Date: Mon, 17 Jun 2024 06:13:07 -0600 Subject: [PATCH 14/27] add js docs --- apps/cf-ai-backend/src/utils/chonker.ts | 3 +++ apps/cf-ai-backend/src/utils/seededRandom.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts index 39d4b458..c63020be 100644 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -1,5 +1,8 @@ import nlp from "compromise"; +/** + * Split text into chunks of specified max size with some overlap for continuity. + */ export default function chunkText( text: string, maxChunkSize: number, diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts index 36a1e4f9..9e315ee8 100644 --- a/apps/cf-ai-backend/src/utils/seededRandom.ts +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -1,5 +1,9 @@ import { MersenneTwister19937, integer } from "random-js"; +/** + * Hashes a string to a 32-bit integer. + * @param {string} seed - The input string to hash. + */ function hashString(seed: string) { let hash = 0; for (let i = 0; i < seed.length; i++) { @@ -10,6 +14,9 @@ function hashString(seed: string) { return hash; } +/** + * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1. + */ export function seededRandom(seed: string) { const seedHash = hashString(seed); const engine = MersenneTwister19937.seed(seedHash); From 957f131c1c2f0567a1a06c572b353d2397edeead Mon Sep 17 00:00:00 2001 From: Dhravya Date: Mon, 17 Jun 2024 20:06:37 -0500 Subject: [PATCH 15/27] vector duplication no longer an issue. support for querying with multiple spaces. --- apps/cf-ai-backend/src/helper.ts | 88 +++++++++++++++++++++++----- apps/cf-ai-backend/src/index.test.ts | 13 ---- apps/cf-ai-backend/src/index.ts | 13 ++-- apps/cf-ai-backend/src/types.ts | 2 +- 4 files changed, 81 insertions(+), 35 deletions(-) delete mode 100644 apps/cf-ai-backend/src/index.test.ts diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 78ff86da..cef781be 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -49,7 +49,7 @@ export async function initQuery( selectedModel = openai.chat("gpt-4o"); break; } - + return { store, model: selectedModel }; } @@ -64,19 +64,46 @@ export async function deleteDocument({ c: Context<{ Bindings: Env }>; store: CloudflareVectorizeStore; }) { - const toBeDeleted = `${url}-${user}`; + const toBeDeleted = `${url}#supermemory-web`; const random = seededRandom(toBeDeleted); const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - await c.env.KV.list({ prefix: uuid }).then(async (keys) => { - for (const key of keys.keys) { - await c.env.KV.delete(key.name); - await store.delete({ ids: [key.name] }); + const allIds = await c.env.KV.list({ prefix: uuid }); + + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + // We don't actually delete document directly, we just remove the user from the metadata. + // If there's no user left, we can delete the document. + const newVectors = vectors.map((vector) => { + delete vector.metadata[`user-${user}`]; + + // Get count of how many users are left + const userCount = Object.keys(vector.metadata).filter((key) => + key.startsWith("user-"), + ).length; + + // If there's no user left, we can delete the document. + // need to make sure that every chunk is deleted otherwise it would be problematic. + if (userCount === 0) { + store.delete({ ids: savedVectorIds }); + void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id))); + return null; + } + + return vector; + }); + + // If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors. + if (newVectors.every((v) => v === null)) { + await c.env.KV.delete(uuid); + } else { + await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null)); } - }); + } } export async function batchCreateChunksAndEmbeddings({ @@ -90,15 +117,44 @@ export async function batchCreateChunksAndEmbeddings({ chunks: string[]; context: Context<{ Bindings: Env }>; }) { - const ourID = `${body.url}/#supermemory-${body.user}`; - - await deleteDocument({ url: body.url, user: body.user, c: context, store }); - + //! NOTE that we use #supermemory-web to ensure that + //! If a user saves it through the extension, we don't want other users to be able to see it. + // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. + // I cannot stress this enough, important for security. + const ourID = `${body.url}#supermemory-web`; const random = seededRandom(ourID); const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); + const allIds = await context.env.KV.list({ prefix: uuid }); + + // If some chunks for that content already exist, we'll just update the metadata to include + // the user. + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + + // Now, we'll update all vector metadatas with one more userId and all spaceIds + const newVectors = vectors.map((vector) => { + vector.metadata = { + ...vector.metadata, + [`user-${body.user}`]: 1, + + // For each space in body, add the spaceId to the vector metadata + ...(body.spaces ?? [])?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), + }; + + return vector; + }); + + await context.env.VECTORIZE_INDEX.upsert(newVectors); + return; + } + for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; const chunkId = `${uuid}-${i}`; @@ -112,11 +168,15 @@ export async function batchCreateChunksAndEmbeddings({ metadata: { title: body.title?.slice(0, 50) ?? "", description: body.description ?? "", - space: body.space ?? "", url: body.url, - user: body.user, type: body.type ?? "page", content: newPageContent, + + [`user-${body.user}`]: 1, + ...body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), }, }, ], @@ -127,6 +187,6 @@ export async function batchCreateChunksAndEmbeddings({ console.log("Docs added: ", docs); - await context.env.KV.put(uuid, ourID); + await context.env.KV.put(chunkId, ourID); } } diff --git a/apps/cf-ai-backend/src/index.test.ts b/apps/cf-ai-backend/src/index.test.ts deleted file mode 100644 index bbf66fb5..00000000 --- a/apps/cf-ai-backend/src/index.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import app from "."; - -// TODO: write more tests -describe("Test the application", () => { - it("Should return 200 response", async () => { - const res = await app.request("http://localhost/"); - expect(res.status).toBe(200); - }), - it("Should return 404 response", async () => { - const res = await app.request("http://localhost/404"); - expect(res.status).toBe(404); - }); -}); diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 75a3b8e8..a4c5cbfd 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -87,7 +87,7 @@ app.post( .min(1, "At least one image is required") .optional(), text: z.string().optional(), - space: z.string().optional(), + spaces: z.array(z.string()).optional(), url: z.string(), user: z.string(), }), @@ -134,7 +134,7 @@ app.post( imageDescriptions.length > 1 ? `A group of ${imageDescriptions.length} images on ${body.url}` : imageDescriptions[0], - space: body.space, + spaces: body.spaces, pageContent: imageDescriptions.join("\n"), title: "Image content from the web", }, @@ -198,7 +198,9 @@ app.post( // Get the AI model maker and vector store const { model, store } = await initQuery(c, query.model); - const filter: VectorizeVectorMetadataFilter = { user: query.user }; + const filter: VectorizeVectorMetadataFilter = { + [`user-${query.user}`]: 1, + }; console.log("Spaces", spaces); // Converting the query to a vector so that we can search for similar vectors @@ -212,7 +214,7 @@ app.post( console.log("space", space); if (!space && spaces.length > 1) { // it's possible for space list to be [undefined] so we only add space filter conditionally - filter.space = space; + filter[`space-${query.user}-${space}`] = 1; } // Because there's no OR operator in the filter, we have to make multiple queries @@ -265,9 +267,6 @@ app.post( dataPoint.id.toString(), ); - // We are getting the content ID back, so that the frontend can show the actual sources properly. - // it IS a lot of DB calls, i completely agree. - // TODO: return metadata value here, so that the frontend doesn't have to re-fetch anything. const storedContent = await Promise.all( idsAsStrings.map(async (id) => await c.env.KV.get(id)), ); diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index 5f6d0583..417d6320 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -43,7 +43,7 @@ export const vectorObj = z.object({ pageContent: z.string(), title: z.string().optional(), description: z.string().optional(), - space: z.string().optional(), + spaces: z.array(z.string()).optional(), url: z.string(), user: z.string(), type: z.string().optional().default("page"), From 066833a753eed90577b6d28f30ada99f63b5906e Mon Sep 17 00:00:00 2001 From: Dhravya Date: Mon, 17 Jun 2024 20:07:59 -0500 Subject: [PATCH 16/27] include all selected spaces in the fetch call to backend --- apps/web/app/actions/doers.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index f94ed8ec..6c7180d9 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -168,8 +168,7 @@ export const createMemory = async (input: { title: metadata.title, description: metadata.description, url: metadata.baseUrl, - // TODO: now, in the vector store, we are only saving the first space. We need to save all spaces. - space: storeToSpaces[0], + spaces: storeToSpaces, user: data.user.id, type, }), From 242cbf721acaebc2f77ce161cef10d2c04261388 Mon Sep 17 00:00:00 2001 From: codetorso Date: Tue, 18 Jun 2024 03:02:18 -0600 Subject: [PATCH 17/27] Add Editor Endpoint --- apps/cf-ai-backend/src/index.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 75a3b8e8..c4ffa6b5 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { Hono } from "hono"; -import { CoreMessage, streamText } from "ai"; +import { CoreMessage, generateText, streamText } from "ai"; import { chatObj, Env, vectorObj } from "./types"; import { batchCreateChunksAndEmbeddings, @@ -331,4 +331,20 @@ app.delete( }, ); +app.get('/api/editorai', zValidator( + "query", + z.object({ + context: z.string(), + request: z.string(), + }), +), async (c)=> { + const { context, request } = c.req.valid("query"); + + const { model } = await initQuery(c); + + const {text} = await generateText({ model, prompt: `${request}-${context}` }); + + return c.json({completion: text}); +}) + export default app; From 6cfb5331ad9bd30017af27d562f8e1fbc0302db5 Mon Sep 17 00:00:00 2001 From: codetorso Date: Tue, 18 Jun 2024 03:31:09 -0600 Subject: [PATCH 18/27] EditorAI integrated! (1/4) --- apps/cf-ai-backend/src/index.ts | 2 +- apps/web/app/(canvas)/canvas/page.tsx | 2 +- .../app/(editor)/components/aigenerate.tsx | 80 +++++++++---------- 3 files changed, 38 insertions(+), 46 deletions(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 35e03ce1..effdf517 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -341,7 +341,7 @@ app.get('/api/editorai', zValidator( const { model } = await initQuery(c); - const {text} = await generateText({ model, prompt: `${request}-${context}` }); + const {text} = await generateText({ model, prompt: `${request}-${context}`, maxTokens: 224 }); return c.json({completion: text}); }) diff --git a/apps/web/app/(canvas)/canvas/page.tsx b/apps/web/app/(canvas)/canvas/page.tsx index 7abfa583..366a4481 100644 --- a/apps/web/app/(canvas)/canvas/page.tsx +++ b/apps/web/app/(canvas)/canvas/page.tsx @@ -18,7 +18,7 @@ function page() { const [fullScreen, setFullScreen] = useState(false); return ( -
+
{setTimeout(()=> setFullScreen(false), 50)}} onCollapse={()=> {setTimeout(()=> setFullScreen(true), 50)}} defaultSize={30} collapsible={true} minSize={22}> diff --git a/apps/web/app/(editor)/components/aigenerate.tsx b/apps/web/app/(editor)/components/aigenerate.tsx index b1c4ccd4..f27fd50f 100644 --- a/apps/web/app/(editor)/components/aigenerate.tsx +++ b/apps/web/app/(editor)/components/aigenerate.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import Magic from "./ui/magic"; import CrazySpinner from "./ui/crazy-spinner"; import Asksvg from "./ui/asksvg"; @@ -9,13 +9,9 @@ import { motion, AnimatePresence } from "framer-motion"; import type { Editor } from "@tiptap/core"; import { useEditor } from "novel"; - function Aigenerate() { const [visible, setVisible] = useState(false); const [generating, setGenerating] = useState(false); - - // generating -> can be converted to false, so we need to make sure the generation gets cancelled - // visible const { editor } = useEditor(); const setGeneratingfn = (v: boolean) => setGenerating(v); @@ -58,8 +54,8 @@ function Aigenerate() { }} className="absolute z-50 top-0" > - -
+ +
); @@ -101,7 +97,7 @@ function ToolBar({ onClick={() => AigenerateContent({ idx, editor, setGeneratingfn }) } - className="absolute select-none inset-0 block h-full w-full rounded-xl bg-[#33393D]" + className="absolute select-none inset-0 block h-full w-full rounded-xl bg-background-light" layoutId="hoverBackground" initial={{ opacity: 0 }} animate={{ @@ -116,7 +112,7 @@ function ToolBar({ )}
- {item} + {item}
))} @@ -135,43 +131,39 @@ async function AigenerateContent({ }) { setGeneratingfn(true); - const {from, to} = editor.view.state.selection; - const content = editor.view.state.selection.content(); - content.content.forEach((v, i)=> { - v.forEach((v, i)=> { - console.log(v.text) - }) + const { from, to } = editor.view.state.selection; + + const slice = editor.state.selection.content(); + const text = editor.storage.markdown.serializer.serialize(slice.content); + + const request = [ + "Translate to hindi written in english, do not write anything else", + "change tone, improve the way be more formal", + "ask, answer the question", + "continue this, maximum 30 characters, do not repeat just continue don't use ... to denote start", + ] + + const res = await fetch("/api/editorai", { + method: "POST", + body: JSON.stringify({ + context: text, + request: request[idx], + }), }) + const {completion}: {completion: string} = await res.json(); + console.log(completion) - const transaction = editor.state.tr - transaction.replaceRange(from, to, content) - - editor.view.dispatch(transaction) - - // console.log(content) - // content.map((v, i)=> console.log(v.content)) - - // const fragment = Fragment.fromArray(content); - - // console.log(fragment) - - // editor.view.state.selection.content().content.append(content) + if (idx === 0 || idx === 1){ + const selectionLength = completion.length + from + editor.chain().focus() + .insertContentAt({from, to}, completion).setTextSelection({from, to: selectionLength}) + .run(); + } else { + const selectionLength = completion.length + to + 1 + editor.chain().focus() + .insertContentAt(to+1, completion).setTextSelection({from, to: selectionLength}) + .run(); + } setGeneratingfn(false); - - - - // const genAI = new GoogleGenerativeAI("AIzaSyDGwJCP9SH5gryyvh65LJ6xTZ0SOdNvzyY"); - // const model = genAI.getGenerativeModel({ model: "gemini-pro"}); - - // const result = (await model.generateContent(`${ty}, ${query}`)).response.text(); - - // .insertContentAt( - // { - // from: from, - // to: to, - // }, - // result, - // ) - // .run(); } From c5361aa24df2cdf50a6189df0fb1493019ecbfc3 Mon Sep 17 00:00:00 2001 From: codetorso Date: Tue, 18 Jun 2024 03:54:20 -0600 Subject: [PATCH 19/27] Create Embeddings for Canvas --- apps/web/app/(canvas)/lib/createEmbeds.ts | 16 ++- apps/web/app/api/editorai/route.ts | 20 ++++ apps/web/app/api/unfirlsite/route.ts | 134 ++++++++++++++++++++++ apps/web/cf-env.d.ts | 5 +- package.json | 1 + 5 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 apps/web/app/api/editorai/route.ts create mode 100644 apps/web/app/api/unfirlsite/route.ts diff --git a/apps/web/app/(canvas)/lib/createEmbeds.ts b/apps/web/app/(canvas)/lib/createEmbeds.ts index 322e697e..53d81533 100644 --- a/apps/web/app/(canvas)/lib/createEmbeds.ts +++ b/apps/web/app/(canvas)/lib/createEmbeds.ts @@ -50,10 +50,18 @@ export default async function createEmbedsFromUrl({url, point, sources, editor}: type: "url", url, }); - const fetchWebsite = await (await fetch(`https://unfurl-bookmark.pruthvirajthinks.workers.dev/?url=${url}`)).json() - if (fetchWebsite.title) bookmarkAsset.props.title = fetchWebsite.title; - if (fetchWebsite.image) bookmarkAsset.props.image = fetchWebsite.image; - if (fetchWebsite.description) bookmarkAsset.props.description = fetchWebsite.description; + const fetchWebsite: { + title?: string; + image?: string; + description?: string; + } = await (await fetch(`/api/unfirlsite?website=${url}`, { + method: "POST" + })).json() + if (bookmarkAsset){ + if (fetchWebsite.title) bookmarkAsset.props.title = fetchWebsite.title; + if (fetchWebsite.image) bookmarkAsset.props.image = fetchWebsite.image; + if (fetchWebsite.description) bookmarkAsset.props.description = fetchWebsite.description; + } if (!bookmarkAsset) throw Error("Could not create an asset"); asset = bookmarkAsset; } catch (e) { diff --git a/apps/web/app/api/editorai/route.ts b/apps/web/app/api/editorai/route.ts new file mode 100644 index 00000000..6ee0aed2 --- /dev/null +++ b/apps/web/app/api/editorai/route.ts @@ -0,0 +1,20 @@ +import type { NextRequest } from "next/server"; +import { ensureAuth } from "../ensureAuth"; + +export const runtime = "edge"; + +export async function POST(request: NextRequest) { + const d = await ensureAuth(request); + if (!d) { + return new Response("Unauthorized", { status: 401 }); + } + const res : {context: string, request: string} = await request.json() + + try { + const response = await fetch(`${process.env.BACKEND_BASE_URL}/api/editorai?context=${res.context}&request=${res.request}`); + const result = await response.json(); + return new Response(JSON.stringify(result)); + } catch (error) { + return new Response(`Error, ${error}`) + } +} \ No newline at end of file diff --git a/apps/web/app/api/unfirlsite/route.ts b/apps/web/app/api/unfirlsite/route.ts new file mode 100644 index 00000000..4b8b4858 --- /dev/null +++ b/apps/web/app/api/unfirlsite/route.ts @@ -0,0 +1,134 @@ +import { load } from 'cheerio' +import { AwsClient } from "aws4fetch"; + +import type { NextRequest } from "next/server"; +import { ensureAuth } from "../ensureAuth"; + +export const runtime = "edge"; + +const r2 = new AwsClient({ + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, +}); + + +export async function POST(request: NextRequest) { + + const d = await ensureAuth(request); + if (!d) { + return new Response("Unauthorized", { status: 401 }); + } + + if ( + !process.env.R2_ACCESS_KEY_ID || + !process.env.R2_ACCOUNT_ID || + !process.env.R2_SECRET_ACCESS_KEY || + !process.env.R2_BUCKET_NAME + ) { + return new Response( + "Missing one or more R2 env variables: R2_ENDPOINT, R2_ACCESS_ID, R2_SECRET_KEY, R2_BUCKET_NAME. To get them, go to the R2 console, create and paste keys in a `.dev.vars` file in the root of this project.", + { status: 500 }, + ); + } + + const website = new URL(request.url).searchParams.get("website"); + + if (!website) { + return new Response("Missing website", { status: 400 }); + } + + const salt = () => Math.floor(Math.random() * 11); + const encodeWebsite = `${encodeURIComponent(website)}${salt()}`; + + try { + // this returns the og image, description and title of website + const response = await unfurl(website); + + if (!response.image){ + return new Response(JSON.stringify(response)) + } + + const imageUrl = await process.env.DEV_IMAGES.get(encodeWebsite) + if (imageUrl){ + return new Response(JSON.stringify({ + image: imageUrl, + title: response.title, + description: response.description, + })) + } + + const res = await fetch(`${response.image}`) + const image = await res.blob(); + + const url = new URL( + `https://${process.env.R2_BUCKET_NAME}.${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com` + ); + + url.pathname = encodeWebsite; + url.searchParams.set("X-Amz-Expires", "3600"); + + const signedPuturl = await r2.sign( + new Request(url, { + method: "PUT", + }), + { + aws: { signQuery: true }, + } + ); + await fetch(signedPuturl.url, { + method: 'PUT', + body: image, + }); + + await process.env.DEV_IMAGES.put(encodeWebsite, `${process.env.R2_PUBLIC_BUCKET_ADDRESS}/${encodeWebsite}`) + + return new Response(JSON.stringify({ + image: `${process.env.R2_PUBLIC_BUCKET_ADDRESS}/${encodeWebsite}`, + title: response.title, + description: response.description, + })); + + } catch (error) { + console.log(error) + return new Response(JSON.stringify({ + status: 500, + error: error, + })) + } + } + +export async function unfurl(url: string) { + const response = await fetch(url) + if (response.status >= 400) { + throw new Error(`Error fetching url: ${response.status}`) + } + const contentType = response.headers.get('content-type') + if (!contentType?.includes('text/html')) { + throw new Error(`Content-type not right: ${contentType}`) + } + + const content = await response.text() + const $ = load(content) + + const og: { [key: string]: string | undefined } = {} + const twitter: { [key: string]: string | undefined } = {} + + // @ts-ignore, it just works so why care of type safety if someone has better way go ahead + $('meta[property^=og:]').each((_, el) => (og[$(el).attr('property')!] = $(el).attr('content'))) + // @ts-ignore + $('meta[name^=twitter:]').each((_, el) => (twitter[$(el).attr('name')!] = $(el).attr('content'))) + + const title = og['og:title'] ?? twitter['twitter:title'] ?? $('title').text() ?? undefined + const description = + og['og:description'] ?? + twitter['twitter:description'] ?? + $('meta[name="description"]').attr('content') ?? + undefined + const image = og['og:image:secure_url'] ?? og['og:image'] ?? twitter['twitter:image'] ?? undefined + + return { + title, + description, + image, + } +} diff --git a/apps/web/cf-env.d.ts b/apps/web/cf-env.d.ts index e98c36cf..be5c991a 100644 --- a/apps/web/cf-env.d.ts +++ b/apps/web/cf-env.d.ts @@ -5,8 +5,9 @@ declare global { GOOGLE_CLIENT_SECRET: string; AUTH_SECRET: string; R2_ENDPOINT: string; - R2_ACCESS_ID: string; - R2_SECRET_KEY: string; + R2_ACCESS_KEY_ID: string; + R2_SECRET_ACCESS_KEY: string; + R2_PUBLIC_BUCKET_ADDRESS: string; R2_BUCKET_NAME: string; BACKEND_SECURITY_KEY: string; BACKEND_BASE_URL: string; diff --git a/package.json b/package.json index dd384010..7ceb7192 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "@tldraw/assets": "^2.2.0", "@types/readline-sync": "^1.4.8", "ai": "^3.1.14", + "aws4fetch": "^1.0.18", "cheerio": "^1.0.0-rc.12", "compromise": "^14.13.0", "drizzle-orm": "^0.30.10", From 09af2ec8f659aab289ad307e4f61abd58c7c6254 Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Tue, 18 Jun 2024 17:59:14 -0500 Subject: [PATCH 20/27] darkmode by default --- packages/tailwind-config/globals.css | 46 +++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/packages/tailwind-config/globals.css b/packages/tailwind-config/globals.css index d845aca4..a77eacbb 100644 --- a/packages/tailwind-config/globals.css +++ b/packages/tailwind-config/globals.css @@ -2,39 +2,37 @@ @tailwind components; @tailwind utilities; -@media (prefers-color-scheme: dark) { - :root { - --foreground: rgba(179, 188, 197, 1); - --foreground-menu: rgba(106, 115, 125, 1); - --background: rgba(23, 27, 31, 1); - --secondary: rgba(31, 36, 40, 1); - --primary: rgba(54, 157, 253, 1); - --border: rgba(51, 57, 67, 1); +:root { + --foreground: rgba(179, 188, 197, 1); + --foreground-menu: rgba(106, 115, 125, 1); + --background: rgba(23, 27, 31, 1); + --secondary: rgba(31, 36, 40, 1); + --primary: rgba(54, 157, 253, 1); + --border: rgba(51, 57, 67, 1); - --card: 0 0% 100%; - --card-foreground: 0 0% 3.9%; + --card: 0 0% 100%; + --card-foreground: 0 0% 3.9%; - --popover: 0 0% 100%; - --popover-foreground: 0 0% 3.9%; + --popover: 0 0% 100%; + --popover-foreground: 0 0% 3.9%; - --primary-foreground: 0 0% 98%; + --primary-foreground: 0 0% 98%; - --secondary-foreground: 0 0% 9%; + --secondary-foreground: 0 0% 9%; - --muted: 0 0% 96.1%; - --muted-foreground: 0 0% 45.1%; + --muted: 0 0% 96.1%; + --muted-foreground: 0 0% 45.1%; - --accent: 0 0% 96.1%; - --accent-foreground: 0 0% 9%; + --accent: 0 0% 96.1%; + --accent-foreground: 0 0% 9%; - --destructive: 0 84.2% 60.2%; - --destructive-foreground: 0 0% 98%; + --destructive: 0 84.2% 60.2%; + --destructive-foreground: 0 0% 98%; - --input: 0 0% 89.8%; - --ring: 0 0% 3.9%; + --input: 0 0% 89.8%; + --ring: 0 0% 3.9%; - --radius: 0.5rem; - } + --radius: 0.5rem; } body { From 62054df2314270a484dc277691b5726969f85c65 Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Tue, 18 Jun 2024 18:38:25 -0500 Subject: [PATCH 21/27] added logic to handle justification properly --- apps/web/app/(dash)/chat/chatWindow.tsx | 55 +++++++++++++++++++++---- packages/shared-types/index.ts | 1 + 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 77c1f32b..17c415e9 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -23,7 +23,6 @@ import { codeLanguageSubset } from "@/lib/constants"; import { z } from "zod"; import { toast } from "sonner"; import Link from "next/link"; -import { sources } from "next/dist/compiled/webpack/webpack"; function ChatWindow({ q, @@ -47,6 +46,20 @@ function ChatWindow({ }, ]); + const removeJustificationFromText = (text: string) => { + // remove everything after the first "" word + const justificationLine = text.indexOf(""); + if (justificationLine !== -1) { + // Add that justification to the last chat message + const lastChatMessage = chatHistory[chatHistory.length - 1]; + if (lastChatMessage) { + lastChatMessage.answer.justification = text.slice(justificationLine); + } + return text.slice(0, justificationLine); + } + return text; + }; + const router = useRouter(); const getAnswer = async (query: string, spaces: string[]) => { @@ -55,7 +68,7 @@ function ChatWindow({ { method: "POST", body: JSON.stringify({ chatHistory }), - }, + } ); // TODO: handle this properly @@ -80,7 +93,7 @@ function ChatWindow({ const lastAnswer = newChatHistory[newChatHistory.length - 1]; if (!lastAnswer) return prevChatHistory; const filteredSourceUrls = new Set( - sourcesParsed.data.metadata.map((source) => source.url), + sourcesParsed.data.metadata.map((source) => source.url) ); const uniqueSources = sourcesParsed.data.metadata.filter((source) => { if (filteredSourceUrls.has(source.url)) { @@ -95,7 +108,7 @@ function ChatWindow({ source: source.url ?? "https://supermemory.ai", content: source.content ?? "No content available", numChunks: sourcesParsed.data.metadata.filter( - (f) => f.url === source.url, + (f) => f.url === source.url ).length, })); return newChatHistory; @@ -129,7 +142,7 @@ function ChatWindow({ if (q.trim().length > 0) { getAnswer( q, - spaces.map((s) => s.id), + spaces.map((s) => s.id) ); setTimeout(() => { setLayout("chat"); @@ -164,7 +177,7 @@ function ChatWindow({ >

{chat.question} @@ -262,10 +275,38 @@ function ChatWindow({ }} className="flex flex-col gap-2" > - {chat.answer.parts.map((part) => part.text).join("")} + {removeJustificationFromText( + chat.answer.parts.map((part) => part.text).join("") + )}

+ + {/* Justification */} + {chat.answer.justification && + chat.answer.justification.length && ( +
0 ? "flex" : "hidden"}`} + > + + + + Justification + + + {chat.answer.justification.length > 0 + ? chat.answer.justification + .replaceAll("", "") + .replaceAll("", "") + : "No justification provided."} + + + +
+ )}
))} diff --git a/packages/shared-types/index.ts b/packages/shared-types/index.ts index b8792369..d3f466e1 100644 --- a/packages/shared-types/index.ts +++ b/packages/shared-types/index.ts @@ -13,6 +13,7 @@ export const ChatHistoryZod = z.object({ numChunks: z.number().optional().default(1), }), ), + justification: z.string().optional(), }), }); From c704541e805e41b001f0715b0436168849fcc95e Mon Sep 17 00:00:00 2001 From: MaheshtheDev Date: Tue, 18 Jun 2024 20:26:15 -0700 Subject: [PATCH 22/27] fix: Cta component background color --- apps/web/app/(landing)/Cta.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/app/(landing)/Cta.tsx b/apps/web/app/(landing)/Cta.tsx index be99bf99..f0f471c2 100644 --- a/apps/web/app/(landing)/Cta.tsx +++ b/apps/web/app/(landing)/Cta.tsx @@ -24,7 +24,7 @@ function Cta() { height={1405} priority draggable="false" - className="absolute z-[-2] hidden select-none rounded-3xl bg-black md:block lg:w-[80%]" + className="absolute z-[-2] hidden select-none rounded-3xl bg-background md:block lg:w-[80%]" />

Your bookmarks are collecting dust. From 770eb99a30e884d4eef8acdcd6556f2e91df7aee Mon Sep 17 00:00:00 2001 From: codetorso Date: Tue, 18 Jun 2024 23:46:14 -0600 Subject: [PATCH 23/27] Drag and Drop in Canvas! --- apps/web/app/(canvas)/canvas.tsx | 79 ++++++++++++++++++----- apps/web/app/(canvas)/dropComponent.tsx | 76 ++++++++++++++++++++++ apps/web/app/(canvas)/enabledComp.tsx | 2 +- apps/web/app/(canvas)/lib/context.tsx | 11 ++++ apps/web/app/(canvas)/lib/createEmbeds.ts | 36 ++++++++++- 5 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 apps/web/app/(canvas)/dropComponent.tsx create mode 100644 apps/web/app/(canvas)/lib/context.tsx diff --git a/apps/web/app/(canvas)/canvas.tsx b/apps/web/app/(canvas)/canvas.tsx index 9ec57d6d..498ab1eb 100644 --- a/apps/web/app/(canvas)/canvas.tsx +++ b/apps/web/app/(canvas)/canvas.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Editor, Tldraw, setUserPreferences, TLStoreWithStatus } from "tldraw"; import { createAssetFromUrl } from "./lib/createAssetUrl"; import "tldraw/tldraw.css"; @@ -7,10 +7,53 @@ import { twitterCardUtil } from "./twitterCard"; import createEmbedsFromUrl from "./lib/createEmbeds"; import { loadRemoteSnapshot } from "./lib/loadSnap"; import { SaveStatus } from "./savesnap"; -import { getAssetUrls } from '@tldraw/assets/selfHosted' -import { memo } from 'react'; +import { getAssetUrls } from "@tldraw/assets/selfHosted"; +import { memo } from "react"; +import DragContext from "./lib/context"; +import DropZone from "./dropComponent"; -export const Canvas = memo(()=>{ +export const Canvas = memo(() => { + const [isDraggingOver, setIsDraggingOver] = useState(false); + const Dragref = useRef(null) + + const handleDragOver = (event: any) => { + event.preventDefault(); + setIsDraggingOver(true); + console.log("entere") + }; + + const handleDragLeave = () => { + setIsDraggingOver(false); + console.log("leaver") + }; + + useEffect(() => { + const divElement = Dragref.current; + if (divElement) { + divElement.addEventListener('dragover', handleDragOver); + divElement.addEventListener('dragleave', handleDragLeave); + } + return () => { + if (divElement) { + divElement.removeEventListener('dragover', handleDragOver); + divElement.removeEventListener('dragleave', handleDragLeave); + } + }; + }, []); + + return ( + +
+ +
+ + ); +}); + +const TldrawComponent =memo(() => { const [storeWithStatus, setStoreWithStatus] = useState({ status: "loading", }); @@ -38,18 +81,22 @@ export const Canvas = memo(()=>{ setUserPreferences({ id: "supermemory", isDarkMode: true }); - const assetUrls = getAssetUrls() + const assetUrls = getAssetUrls(); return ( - -
- -
- +
+ +
+ +
+ + +
); }) diff --git a/apps/web/app/(canvas)/dropComponent.tsx b/apps/web/app/(canvas)/dropComponent.tsx new file mode 100644 index 00000000..03a32358 --- /dev/null +++ b/apps/web/app/(canvas)/dropComponent.tsx @@ -0,0 +1,76 @@ +import React, { useRef, useCallback, useEffect, useContext } from "react"; +import { useEditor } from "tldraw"; +import DragContext, { DragContextType } from "./lib/context"; +import { handleExternalDroppedContent } from "./lib/createEmbeds"; + +const stripHtmlTags = (html: string): string => { + const div = document.createElement("div"); + div.innerHTML = html; + return div.textContent || div.innerText || ""; +}; + +const useDrag = (): DragContextType => { + const context = useContext(DragContext); + if (!context) { + throw new Error('useCounter must be used within a CounterProvider'); + } + return context; +}; + + +function DropZone() { + const dropRef = useRef(null); + const {isDraggingOver, setIsDraggingOver} = useDrag(); + + const editor = useEditor(); + + const handleDrop = useCallback((event: React.DragEvent) => { + event.preventDefault(); + setIsDraggingOver(false); + const dt = event.dataTransfer; + const items = dt.items; + + for (let i = 0; i < items.length; i++) { + if (items[i]!.kind === "file" && items[i]!.type.startsWith("image/")) { + const file = items[i]!.getAsFile(); + if (file) { + const reader = new FileReader(); + reader.onload = (e) => { + if (e.target) { + // setDroppedImage(e.target.result as string); + } + }; + reader.readAsDataURL(file); + } + } else if (items[i]!.kind === "string") { + items[i]!.getAsString((data) => { + const cleanText = stripHtmlTags(data); + handleExternalDroppedContent({editor,text:cleanText}) + }); + } + } + }, []); + + useEffect(() => { + const divElement = dropRef.current; + if (divElement) { + // @ts-ignore + divElement.addEventListener("drop", handleDrop); + } + return () => { + if (divElement) { + // @ts-ignore + divElement.removeEventListener("drop", handleDrop); + } + }; + }, []); + + return ( +
+ ); +} + +export default DropZone; diff --git a/apps/web/app/(canvas)/enabledComp.tsx b/apps/web/app/(canvas)/enabledComp.tsx index 5dbe6ee7..85811b82 100644 --- a/apps/web/app/(canvas)/enabledComp.tsx +++ b/apps/web/app/(canvas)/enabledComp.tsx @@ -7,12 +7,12 @@ export const components: Partial = { TopPanel: null, DebugPanel: null, DebugMenu: null, + PageMenu: null, // Minimap: null, // ContextMenu: null, // HelpMenu: null, // ZoomMenu: null, // StylePanel: null, - // PageMenu: null, // NavigationPanel: null, // Toolbar: null, // KeyboardShortcutsDialog: null, diff --git a/apps/web/app/(canvas)/lib/context.tsx b/apps/web/app/(canvas)/lib/context.tsx new file mode 100644 index 00000000..36a106cf --- /dev/null +++ b/apps/web/app/(canvas)/lib/context.tsx @@ -0,0 +1,11 @@ +import { createContext } from 'react'; + +export interface DragContextType { + isDraggingOver: boolean; + setIsDraggingOver: React.Dispatch>; +} + + +const DragContext = createContext(undefined); + +export default DragContext; \ No newline at end of file diff --git a/apps/web/app/(canvas)/lib/createEmbeds.ts b/apps/web/app/(canvas)/lib/createEmbeds.ts index 53d81533..0db3c71b 100644 --- a/apps/web/app/(canvas)/lib/createEmbeds.ts +++ b/apps/web/app/(canvas)/lib/createEmbeds.ts @@ -2,8 +2,8 @@ import { AssetRecordType, Editor, TLAsset, TLAssetId, TLBookmarkShape, TLExterna export default async function createEmbedsFromUrl({url, point, sources, editor}: { url: string - point: VecLike | undefined - sources: TLExternalContentSource[] | undefined + point?: VecLike | undefined + sources?: TLExternalContentSource[] | undefined editor: Editor }){ @@ -87,6 +87,38 @@ export default async function createEmbedsFromUrl({url, point, sources, editor}: }); } +function isURL(str: string) { + try { + new URL(str); + return true; + } catch { + return false; + } +} + + +export function handleExternalDroppedContent({text, editor}: {text:string, editor: Editor}){ + const position = editor.inputs.shiftKey + ? editor.inputs.currentPagePoint + : editor.getViewportPageBounds().center; + + if (isURL(text)){ + createEmbedsFromUrl({editor, url: text}) + } else{ + editor.createShape({ + type: "text", + x: position.x - 75, + y: position.y - 75, + props: { + text: text, + size: "s", + textAlign: "start", + }, + }); + + } +} + function centerSelectionAroundPoint(editor: Editor, position: VecLike) { // Re-position shapes so that the center of the group is at the provided point const viewportPageBounds = editor.getViewportPageBounds() From f28f566473d1087afa4fc1b8a51c0a811060b125 Mon Sep 17 00:00:00 2001 From: codetorso Date: Wed, 19 Jun 2024 02:12:47 -0600 Subject: [PATCH 24/27] Improve code, failed attempt at Streaming text --- apps/cf-ai-backend/src/index.ts | 9 +-- apps/web/app/(dash)/chat/chatWindow.tsx | 3 - .../app/(editor)/components/aigenerate.tsx | 68 ++++++++++++++----- apps/web/app/api/editorai/route.ts | 13 ++-- 4 files changed, 64 insertions(+), 29 deletions(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index effdf517..26a9fa48 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -338,12 +338,13 @@ app.get('/api/editorai', zValidator( }), ), async (c)=> { const { context, request } = c.req.valid("query"); - const { model } = await initQuery(c); - const {text} = await generateText({ model, prompt: `${request}-${context}`, maxTokens: 224 }); - - return c.json({completion: text}); + const response = await streamText({ model, prompt: `${request}-${context}`, maxTokens: 224 }); + + const r = response.toTextStreamResponse(); + + return r; }) export default app; diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 17c415e9..6189b874 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -121,7 +121,6 @@ function ChatWindow({ const reader = resp.body?.getReader(); let done = false; - let result = ""; while (!done && reader) { const { value, done: d } = await reader.read(); done = d; @@ -134,8 +133,6 @@ function ChatWindow({ return newChatHistory; }); } - - console.log(result); }; useEffect(() => { diff --git a/apps/web/app/(editor)/components/aigenerate.tsx b/apps/web/app/(editor)/components/aigenerate.tsx index f27fd50f..c5f6f2c1 100644 --- a/apps/web/app/(editor)/components/aigenerate.tsx +++ b/apps/web/app/(editor)/components/aigenerate.tsx @@ -97,7 +97,7 @@ function ToolBar({ onClick={() => AigenerateContent({ idx, editor, setGeneratingfn }) } - className="absolute select-none inset-0 block h-full w-full rounded-xl bg-background-light" + className="absolute select-none inset-0 block h-full w-full rounded-xl bg-[#33393D]" layoutId="hoverBackground" initial={{ opacity: 0 }} animate={{ @@ -140,30 +140,66 @@ async function AigenerateContent({ "Translate to hindi written in english, do not write anything else", "change tone, improve the way be more formal", "ask, answer the question", - "continue this, maximum 30 characters, do not repeat just continue don't use ... to denote start", + "continue this, minimum 80 characters, do not repeat just continue don't use ... to denote start", ] - const res = await fetch("/api/editorai", { + const resp = await fetch("/api/editorai", { method: "POST", body: JSON.stringify({ context: text, request: request[idx], }), - }) - const {completion}: {completion: string} = await res.json(); - console.log(completion) + }); - if (idx === 0 || idx === 1){ - const selectionLength = completion.length + from - editor.chain().focus() - .insertContentAt({from, to}, completion).setTextSelection({from, to: selectionLength}) - .run(); - } else { - const selectionLength = completion.length + to + 1 - editor.chain().focus() - .insertContentAt(to+1, completion).setTextSelection({from, to: selectionLength}) - .run(); + if (!resp.body) { + console.error("No response body"); + return; } + const reader = resp.body.getReader(); + // const decoder = new TextDecoder(); + let done = false; + let position = to; + + while (!done) { + const { value, done: readerDone } = await reader.read(); + done = readerDone; + + if (value) { + const chunk = new TextDecoder().decode(value) + // decoder.decode(value, { stream: true }); + console.log(chunk); + // editor.chain().focus().insertContentAt(position + 1, chunk).run(); + position += chunk.length + } + } + console.log("Stream complete"); + + + // const reader = resp.body?.getReader(); + // let done = false; + // let position = from; + // while (!done && reader) { + // const { value, done: d } = await reader.read(); + // done = d; + + // const cont = new TextDecoder().decode(value) + // console.log(cont); + // + // } + // const {completion}: {completion: string} = await res.json(); + // console.log(completion) + + // if (idx === 0 || idx === 1){ + // const selectionLength = completion.length + from + // editor.chain().focus() + // .insertContentAt({from, to}, completion).setTextSelection({from, to: selectionLength}) + // .run(); + // } else { + // const selectionLength = completion.length + to + 1 + // editor.chain().focus() + // .insertContentAt(to+1, completion).setTextSelection({from, to: selectionLength}) + // .run(); + // } setGeneratingfn(false); } diff --git a/apps/web/app/api/editorai/route.ts b/apps/web/app/api/editorai/route.ts index 6ee0aed2..43d8eb64 100644 --- a/apps/web/app/api/editorai/route.ts +++ b/apps/web/app/api/editorai/route.ts @@ -4,16 +4,17 @@ import { ensureAuth } from "../ensureAuth"; export const runtime = "edge"; export async function POST(request: NextRequest) { - const d = await ensureAuth(request); - if (!d) { - return new Response("Unauthorized", { status: 401 }); - } + // const d = await ensureAuth(request); + // if (!d) { + // return new Response("Unauthorized", { status: 401 }); + // } const res : {context: string, request: string} = await request.json() try { const response = await fetch(`${process.env.BACKEND_BASE_URL}/api/editorai?context=${res.context}&request=${res.request}`); - const result = await response.json(); - return new Response(JSON.stringify(result)); + return new Response(response.body, { status: 200 }); + // const result = await response.json(); + // return new Response(JSON.stringify(result)); } catch (error) { return new Response(`Error, ${error}`) } From a7cca293a66e7b042178fa54f182cb887c3e072f Mon Sep 17 00:00:00 2001 From: codetorso Date: Wed, 19 Jun 2024 08:37:30 -0600 Subject: [PATCH 25/27] Another Failed Attempt at streaming --- apps/cf-ai-backend/src/index.ts | 5 +- .../app/(editor)/components/aigenerate.tsx | 65 ++++++++++--------- apps/web/app/api/editorai/route.ts | 17 +++-- 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 26a9fa48..e89d170c 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -330,6 +330,7 @@ app.delete( }, ); +// ERROR #1 - this is the api that the editor uses, it is just a scrape off of /api/chat so you may check that out app.get('/api/editorai', zValidator( "query", z.object({ @@ -342,9 +343,7 @@ app.get('/api/editorai', zValidator( const response = await streamText({ model, prompt: `${request}-${context}`, maxTokens: 224 }); - const r = response.toTextStreamResponse(); - - return r; + return response.toTextStreamResponse(); }) export default app; diff --git a/apps/web/app/(editor)/components/aigenerate.tsx b/apps/web/app/(editor)/components/aigenerate.tsx index c5f6f2c1..c9ba657b 100644 --- a/apps/web/app/(editor)/components/aigenerate.tsx +++ b/apps/web/app/(editor)/components/aigenerate.tsx @@ -8,6 +8,8 @@ import Autocompletesvg from "./ui/autocompletesvg"; import { motion, AnimatePresence } from "framer-motion"; import type { Editor } from "@tiptap/core"; import { useEditor } from "novel"; +import { NodeSelection } from 'prosemirror-state' + function Aigenerate() { const [visible, setVisible] = useState(false); @@ -143,6 +145,7 @@ async function AigenerateContent({ "continue this, minimum 80 characters, do not repeat just continue don't use ... to denote start", ] + // ERROR #3 - This is where we call the ai generate api const resp = await fetch("/api/editorai", { method: "POST", body: JSON.stringify({ @@ -151,30 +154,41 @@ async function AigenerateContent({ }), }); - if (!resp.body) { - console.error("No response body"); - return; - } - const reader = resp.body.getReader(); - // const decoder = new TextDecoder(); + // this is the exact replica of your chatwindow code, I have + // 2 more versions of these code commented below, but they also dont work + const reader = resp.body?.getReader(); let done = false; - let position = to; + while (!done && reader) { + const { value, done: d } = await reader.read(); + done = d; - while (!done) { - const { value, done: readerDone } = await reader.read(); - done = readerDone; - - if (value) { - const chunk = new TextDecoder().decode(value) - // decoder.decode(value, { stream: true }); - console.log(chunk); - // editor.chain().focus().insertContentAt(position + 1, chunk).run(); - position += chunk.length - } + console.log(new TextDecoder().decode(value)) } - console.log("Stream complete"); + + // 2nd Method + // if (!resp.body) { + // console.error("No response body"); + // return; + // } + // const reader = resp.body.getReader(); + // const decoder = new TextDecoder(); + // let done = false; + // let position = to; + + // while (!done) { + // const { value, done: readerDone } = await reader.read(); + // done = readerDone; + // if (value) { + // const chunk = decoder.decode(value, { stream: true }); + // console.log(chunk); + // editor.chain().focus().insertContentAt(position + 1, chunk).run(); + // position += chunk.length; + // } + // } + // console.log("Stream complete"); + // 3rd method // const reader = resp.body?.getReader(); // let done = false; // let position = from; @@ -184,22 +198,9 @@ async function AigenerateContent({ // const cont = new TextDecoder().decode(value) // console.log(cont); - // // } // const {completion}: {completion: string} = await res.json(); // console.log(completion) - // if (idx === 0 || idx === 1){ - // const selectionLength = completion.length + from - // editor.chain().focus() - // .insertContentAt({from, to}, completion).setTextSelection({from, to: selectionLength}) - // .run(); - // } else { - // const selectionLength = completion.length + to + 1 - // editor.chain().focus() - // .insertContentAt(to+1, completion).setTextSelection({from, to: selectionLength}) - // .run(); - // } - setGeneratingfn(false); } diff --git a/apps/web/app/api/editorai/route.ts b/apps/web/app/api/editorai/route.ts index 43d8eb64..5e1fbf0c 100644 --- a/apps/web/app/api/editorai/route.ts +++ b/apps/web/app/api/editorai/route.ts @@ -3,6 +3,8 @@ import { ensureAuth } from "../ensureAuth"; export const runtime = "edge"; +// ERROR #2 - This the the next function that calls the backend, I sometimes think this is redundency, but whatever +// I have commented the auth code, It should not work in development, but it still does sometimes export async function POST(request: NextRequest) { // const d = await ensureAuth(request); // if (!d) { @@ -11,10 +13,17 @@ export async function POST(request: NextRequest) { const res : {context: string, request: string} = await request.json() try { - const response = await fetch(`${process.env.BACKEND_BASE_URL}/api/editorai?context=${res.context}&request=${res.request}`); - return new Response(response.body, { status: 200 }); - // const result = await response.json(); - // return new Response(JSON.stringify(result)); + const resp = await fetch(`${process.env.BACKEND_BASE_URL}/api/editorai?context=${res.context}&request=${res.request}`); + // this just checks if there are erros I am keeping it commented for you to better understand the important pieces + // if (resp.status !== 200 || !resp.ok) { + // const errorData = await resp.text(); + // console.log(errorData); + // return new Response( + // JSON.stringify({ message: "Error in CF function", error: errorData }), + // { status: resp.status }, + // ); + // } + return new Response(resp.body, { status: 200 }); } catch (error) { return new Response(`Error, ${error}`) } From e4c6c1aab001be6cbb9473f276002c36f643c9b6 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Wed, 19 Jun 2024 20:04:15 -0500 Subject: [PATCH 26/27] works --- apps/cf-ai-backend/package.json | 2 +- .../app/(editor)/components/aigenerate.tsx | 78 +++++++------------ apps/web/app/(editor)/editor.tsx | 5 +- 3 files changed, 33 insertions(+), 52 deletions(-) diff --git a/apps/cf-ai-backend/package.json b/apps/cf-ai-backend/package.json index 480f9601..78353e08 100644 --- a/apps/cf-ai-backend/package.json +++ b/apps/cf-ai-backend/package.json @@ -6,7 +6,7 @@ "scripts": { "test": "jest --verbose", "deploy": "wrangler deploy", - "dev": "wrangler dev", + "dev": "wrangler dev --remote --port 8686", "start": "wrangler dev", "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" }, diff --git a/apps/web/app/(editor)/components/aigenerate.tsx b/apps/web/app/(editor)/components/aigenerate.tsx index c9ba657b..de9b2a3f 100644 --- a/apps/web/app/(editor)/components/aigenerate.tsx +++ b/apps/web/app/(editor)/components/aigenerate.tsx @@ -8,8 +8,7 @@ import Autocompletesvg from "./ui/autocompletesvg"; import { motion, AnimatePresence } from "framer-motion"; import type { Editor } from "@tiptap/core"; import { useEditor } from "novel"; -import { NodeSelection } from 'prosemirror-state' - +import { NodeSelection } from "prosemirror-state"; function Aigenerate() { const [visible, setVisible] = useState(false); @@ -56,6 +55,7 @@ function Aigenerate() { }} className="absolute z-50 top-0" > + {/* TODO: handle Editor not initalised, maybe with a loading state. */}
@@ -66,10 +66,22 @@ function Aigenerate() { export default Aigenerate; const options = [ - <>Translate, - <>Change Tone, - <>Ask Gemini, - <>Auto Complete + <> + + Translate + , + <> + + Change Tone + , + <> + + Ask Gemini + , + <> + + Auto Complete + , ]; function ToolBar({ @@ -134,7 +146,7 @@ async function AigenerateContent({ setGeneratingfn(true); const { from, to } = editor.view.state.selection; - + const slice = editor.state.selection.content(); const text = editor.storage.markdown.serializer.serialize(slice.content); @@ -143,9 +155,8 @@ async function AigenerateContent({ "change tone, improve the way be more formal", "ask, answer the question", "continue this, minimum 80 characters, do not repeat just continue don't use ... to denote start", - ] + ]; - // ERROR #3 - This is where we call the ai generate api const resp = await fetch("/api/editorai", { method: "POST", body: JSON.stringify({ @@ -154,53 +165,22 @@ async function AigenerateContent({ }), }); - // this is the exact replica of your chatwindow code, I have - // 2 more versions of these code commented below, but they also dont work const reader = resp.body?.getReader(); let done = false; + let position = to; while (!done && reader) { const { value, done: d } = await reader.read(); done = d; - console.log(new TextDecoder().decode(value)) + const decoded = new TextDecoder().decode(value); + console.log(decoded); + editor + .chain() + .focus() + .insertContentAt(position + 1, decoded) + .run(); + position += decoded.length; } - // 2nd Method - // if (!resp.body) { - // console.error("No response body"); - // return; - // } - // const reader = resp.body.getReader(); - // const decoder = new TextDecoder(); - // let done = false; - // let position = to; - - // while (!done) { - // const { value, done: readerDone } = await reader.read(); - // done = readerDone; - // if (value) { - // const chunk = decoder.decode(value, { stream: true }); - // console.log(chunk); - // editor.chain().focus().insertContentAt(position + 1, chunk).run(); - // position += chunk.length; - // } - // } - // console.log("Stream complete"); - - - // 3rd method - // const reader = resp.body?.getReader(); - // let done = false; - // let position = from; - // while (!done && reader) { - // const { value, done: d } = await reader.read(); - // done = d; - - // const cont = new TextDecoder().decode(value) - // console.log(cont); - // } - // const {completion}: {completion: string} = await res.json(); - // console.log(completion) - setGeneratingfn(false); } diff --git a/apps/web/app/(editor)/editor.tsx b/apps/web/app/(editor)/editor.tsx index 5b4a60ce..f7f9a098 100644 --- a/apps/web/app/(editor)/editor.tsx +++ b/apps/web/app/(editor)/editor.tsx @@ -15,19 +15,20 @@ import Topbar from "./components/topbar"; const Editor = () => { const [initialContent, setInitialContent] = useState( - null + null, ); const [saveStatus, setSaveStatus] = useState("Saved"); const [charsCount, setCharsCount] = useState(); const [visible, setVisible] = useState(true); useEffect(() => { + if (typeof window === "undefined") return; const content = window.localStorage.getItem("novel-content"); if (content) setInitialContent(JSON.parse(content)); else setInitialContent(defaultEditorContent); }, []); - if (!initialContent) return null; + if (!initialContent) return <>Loading...; return (
From 074ea24565395fc7c2be5e32c25ca2448541b646 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Wed, 19 Jun 2024 22:34:21 -0500 Subject: [PATCH 27/27] added multi-turn conversations --- apps/web/app/(dash)/chat/chatWindow.tsx | 80 +++++++++++++++------ apps/web/app/(dash)/home/page.tsx | 37 +++++++--- apps/web/app/(dash)/home/queryinput.tsx | 96 ++++++++++++++----------- 3 files changed, 141 insertions(+), 72 deletions(-) diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 6189b874..23f49554 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -1,7 +1,7 @@ "use client"; import { AnimatePresence } from "framer-motion"; -import React, { useEffect, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import QueryInput from "../home/queryinput"; import { cn } from "@repo/ui/lib/utils"; import { motion } from "framer-motion"; @@ -36,15 +36,12 @@ function ChatWindow({ { question: q, answer: { - parts: [ - // { - // text: `It seems like there might be a typo in your question. Could you please clarify or provide more context? If you meant "interesting," please let me know what specific information or topic you find interesting, and I can help you with that.`, - // }, - ], + parts: [], sources: [], }, }, ]); + const [isAutoScroll, setIsAutoScroll] = useState(true); const removeJustificationFromText = (text: string) => { // remove everything after the first "" word @@ -68,7 +65,7 @@ function ChatWindow({ { method: "POST", body: JSON.stringify({ chatHistory }), - } + }, ); // TODO: handle this properly @@ -89,11 +86,15 @@ function ChatWindow({ } setChatHistory((prevChatHistory) => { + window.scrollTo({ + top: document.documentElement.scrollHeight, + behavior: "smooth", + }); const newChatHistory = [...prevChatHistory]; const lastAnswer = newChatHistory[newChatHistory.length - 1]; if (!lastAnswer) return prevChatHistory; const filteredSourceUrls = new Set( - sourcesParsed.data.metadata.map((source) => source.url) + sourcesParsed.data.metadata.map((source) => source.url), ); const uniqueSources = sourcesParsed.data.metadata.filter((source) => { if (filteredSourceUrls.has(source.url)) { @@ -106,9 +107,9 @@ function ChatWindow({ title: source.title ?? "Untitled", type: source.type ?? "page", source: source.url ?? "https://supermemory.ai", - content: source.content ?? "No content available", + content: source.description ?? "No content available", numChunks: sourcesParsed.data.metadata.filter( - (f) => f.url === source.url + (f) => f.url === source.url, ).length, })); return newChatHistory; @@ -129,7 +130,16 @@ function ChatWindow({ const newChatHistory = [...prevChatHistory]; const lastAnswer = newChatHistory[newChatHistory.length - 1]; if (!lastAnswer) return prevChatHistory; - lastAnswer.answer.parts.push({ text: new TextDecoder().decode(value) }); + const txt = new TextDecoder().decode(value); + + if (isAutoScroll) { + window.scrollTo({ + top: document.documentElement.scrollHeight, + behavior: "smooth", + }); + } + + lastAnswer.answer.parts.push({ text: txt }); return newChatHistory; }); } @@ -137,13 +147,11 @@ function ChatWindow({ useEffect(() => { if (q.trim().length > 0) { + setLayout("chat"); getAnswer( q, - spaces.map((s) => s.id) + spaces.map((s) => s.id), ); - setTimeout(() => { - setLayout("chat"); - }, 300); } else { router.push("/home"); } @@ -159,22 +167,27 @@ function ChatWindow({ className="max-w-3xl h-full justify-center items-center flex mx-auto w-full flex-col" >
- + {}} + initialQuery={q} + initialSpaces={[]} + disabled + />
) : (
{chatHistory.map((chat, idx) => (

{chat.question} @@ -273,7 +286,7 @@ function ChatWindow({ className="flex flex-col gap-2" > {removeJustificationFromText( - chat.answer.parts.map((part) => part.text).join("") + chat.answer.parts.map((part) => part.text).join(""), )}

@@ -307,6 +320,33 @@ function ChatWindow({
))} + +
+ { + setChatHistory((prevChatHistory) => { + return [ + ...prevChatHistory, + { + question: q, + answer: { + parts: [], + sources: [], + }, + }, + ]; + }); + await getAnswer( + q, + spaces.map((s) => `${s.id}`), + ); + }} + /> +
)} diff --git a/apps/web/app/(dash)/home/page.tsx b/apps/web/app/(dash)/home/page.tsx index 55f2928e..6fe26513 100644 --- a/apps/web/app/(dash)/home/page.tsx +++ b/apps/web/app/(dash)/home/page.tsx @@ -1,11 +1,12 @@ -import React from "react"; -import Menu from "../menu"; -import Header from "../header"; +"use client"; + +import React, { useEffect, useState } from "react"; import QueryInput from "./queryinput"; import { homeSearchParamsCache } from "@/lib/searchParams"; import { getSpaces } from "@/app/actions/fetchers"; +import { useRouter } from "next/navigation"; -async function Page({ +function Page({ searchParams, }: { searchParams: Record; @@ -13,12 +14,18 @@ async function Page({ // TODO: use this to show a welcome page/modal const { firstTime } = homeSearchParamsCache.parse(searchParams); - let spaces = await getSpaces(); + const [spaces, setSpaces] = useState<{ id: number; name: string }[]>([]); - if (!spaces.success) { - // TODO: handle this error properly. - spaces.data = []; - } + useEffect(() => { + getSpaces().then((res) => { + if (res.success && res.data) { + setSpaces(res.data); + } + // TODO: HANDLE ERROR + }); + }, []); + + const { push } = useRouter(); return (
@@ -26,7 +33,17 @@ async function Page({ {/*
hi {firstTime ? 'first time' : ''}
*/}
- + { + const newQ = + "/chat?q=" + + encodeURI(q) + + (spaces ? "&spaces=" + JSON.stringify(spaces) : ""); + + push(newQ); + }} + initialSpaces={spaces} + />
); diff --git a/apps/web/app/(dash)/home/queryinput.tsx b/apps/web/app/(dash)/home/queryinput.tsx index d0c27b8d..ce45e36b 100644 --- a/apps/web/app/(dash)/home/queryinput.tsx +++ b/apps/web/app/(dash)/home/queryinput.tsx @@ -12,6 +12,9 @@ function QueryInput({ initialQuery = "", initialSpaces = [], disabled = false, + className, + mini = false, + handleSubmit, }: { initialQuery?: string; initialSpaces?: { @@ -19,32 +22,14 @@ function QueryInput({ name: string; }[]; disabled?: boolean; + className?: string; + mini?: boolean; + handleSubmit: (q: string, spaces: { id: number; name: string }[]) => void; }) { const [q, setQ] = useState(initialQuery); const [selectedSpaces, setSelectedSpaces] = useState([]); - const { push } = useRouter(); - - const parseQ = () => { - // preparedSpaces is list of spaces selected by user, with id and name - const preparedSpaces = initialSpaces - .filter((x) => selectedSpaces.includes(x.id)) - .map((x) => { - return { - id: x.id, - name: x.name, - }; - }); - - const newQ = - "/chat?q=" + - encodeURI(q) + - (selectedSpaces ? "&spaces=" + JSON.stringify(preparedSpaces) : ""); - - return newQ; - }; - const options = useMemo( () => initialSpaces.map((x) => ({ @@ -54,21 +39,43 @@ function QueryInput({ [initialSpaces], ); + const preparedSpaces = useMemo( + () => + initialSpaces + .filter((x) => selectedSpaces.includes(x.id)) + .map((x) => { + return { + id: x.id, + name: x.name, + }; + }), + [selectedSpaces, initialSpaces], + ); + return ( -
-
+
+
{/* input and action button */} -
push(parseQ())} className="flex gap-4 p-3"> + { + handleSubmit(q, preparedSpaces); + setQ(""); + }} + className="flex gap-4 p-3" + >