From 0d4c2e062f223710bb23032a6b5eb38a3e22bd27 Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Mon, 22 Jul 2024 21:37:01 -0500 Subject: [PATCH] fix loaders, allow larger websites --- apps/cf-ai-backend/src/index.ts | 35 +++++--- .../cf-ai-backend/src/utils/OpenAIEmbedder.ts | 1 + apps/web/app/(dash)/home/history.tsx | 89 +++++++++---------- apps/web/app/(dash)/home/page.tsx | 4 +- apps/web/app/(dash)/home/queryinput.tsx | 23 ++++- apps/web/app/(dash)/menu.tsx | 2 +- apps/web/app/(landing)/Hero.tsx | 12 ++- packages/ui/shadcn/switch.tsx | 2 +- 8 files changed, 99 insertions(+), 69 deletions(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 413d6d63..13a33536 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -69,21 +69,32 @@ app.post("/api/add", zValidator("json", vectorObj), async (c) => { const newPageContent = body.pageContent?.replace(/.*?<\/raw>/g, ""); const chunks = chunkText(newPageContent, 1536); - if (chunks.length > 20) { - return c.json({ - status: "error", - message: - "We are unable to process documents this size just yet, try something smaller", + + const chunksOf20 = chunks.reduce((acc, chunk, index) => { + if (index % 20 === 0) { + acc.push([chunk]); + } else { + acc[acc.length - 1].push(chunk); + } + return acc; + }, [] as string[][]); + + const accumChunkedInputs = []; + + const promises = chunksOf20.map(async (chunkGroup) => { + const chunkedInput = await batchCreateChunksAndEmbeddings({ + store, + body, + chunks: chunkGroup, + context: c, }); - } - const chunkedInput = await batchCreateChunksAndEmbeddings({ - store, - body, - chunks: chunks, - context: c, + + accumChunkedInputs.push(chunkedInput); }); - return c.json({ status: "ok", chunkedInput }); + await Promise.all(promises); + + return c.json({ status: "ok", chunkedInput: accumChunkedInputs }); }); app.post( diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts index 4a6ff593..41d0802f 100644 --- a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts +++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts @@ -50,6 +50,7 @@ export class OpenAIEmbeddings { const json = zodTypeExpected.safeParse(data); if (!json.success) { + console.log(data); throw new Error("Invalid response from OpenAI: " + json.error.message); } diff --git a/apps/web/app/(dash)/home/history.tsx b/apps/web/app/(dash)/home/history.tsx index 3d8d5a28..551197d8 100644 --- a/apps/web/app/(dash)/home/history.tsx +++ b/apps/web/app/(dash)/home/history.tsx @@ -1,53 +1,50 @@ -import { getChatHistory } from '@repo/web/app/actions/fetchers'; -import { ArrowLongRightIcon } from '@heroicons/react/24/outline'; -import { Skeleton } from '@repo/ui/shadcn/skeleton'; -import Link from 'next/link'; -import { memo, useEffect, useState } from 'react'; -import { motion } from 'framer-motion'; -import { chatThreads } from '@/server/db/schema'; +import { getChatHistory } from "@repo/web/app/actions/fetchers"; +import { ArrowLongRightIcon } from "@heroicons/react/24/outline"; +import { Skeleton } from "@repo/ui/shadcn/skeleton"; +import Link from "next/link"; +import { memo, useEffect, useState } from "react"; +import { motion } from "framer-motion"; +import { chatThreads } from "@/server/db/schema"; const History = memo(() => { - const [chatThreads_, setChatThreads] = useState< - (typeof chatThreads.$inferSelect)[] | null - >(null); + const [chatThreads_, setChatThreads] = useState< + (typeof chatThreads.$inferSelect)[] | null + >(null); - useEffect(() => { - (async () => { - const chatThreads = await getChatHistory(); - if (!chatThreads.success || !chatThreads.data) { - console.error(chatThreads.error); - return; - } - setChatThreads(chatThreads.data.reverse().slice(0, 3)); - })(); - }, []); + useEffect(() => { + (async () => { + const chatThreads = await getChatHistory(); + if (!chatThreads.success || !chatThreads.data) { + console.error(chatThreads.error); + return; + } + setChatThreads(chatThreads.data.reverse().slice(0, 3)); + })(); + }, []); - if (!chatThreads) { - return ( - <> - - - - - ); - } - - return ( - - ); + return ( + + ); }); export default History; diff --git a/apps/web/app/(dash)/home/page.tsx b/apps/web/app/(dash)/home/page.tsx index cc1856b4..53b6cd33 100644 --- a/apps/web/app/(dash)/home/page.tsx +++ b/apps/web/app/(dash)/home/page.tsx @@ -91,7 +91,7 @@ function Page({ searchParams }: { searchParams: Record }) { ...slap, transition: { ...slap.transition, delay: 0.2 }, }} - className="text-center mx-auto bg-[linear-gradient(180deg,_#FFF_0%,_rgba(255,_255,_255,_0.00)_202.08%)] bg-clip-text text-4xl tracking-tighter text-transparent md:text-5xl" + className="text-center mx-auto bg-[linear-gradient(180deg,_#FFF_0%,_rgba(255,_255,_255,_0.00)_202.08%)] bg-clip-text text-4xl tracking-tighter text-transparent md:text-5xl pb-2" > Ask your{" "} @@ -99,7 +99,7 @@ function Page({ searchParams }: { searchParams: Record }) { -
+
([]); @@ -72,9 +76,22 @@ function QueryInput({ setSelectedSpaces={setSelectedSpaces} initialSpaces={initialSpaces || []} /> - +
+
+ + setProMode((prev) => !prev)} + id="pro-mode" + about="Pro mode" + /> +
+ +
diff --git a/apps/web/app/(dash)/menu.tsx b/apps/web/app/(dash)/menu.tsx index 396623ff..34ef3b2d 100644 --- a/apps/web/app/(dash)/menu.tsx +++ b/apps/web/app/(dash)/menu.tsx @@ -334,7 +334,7 @@ function Menu() { />

Add

- {menuItems.map((item) => ( + {menuItems.slice(1, 2).map((item) => (
-
(({ className, ...props }, ref) => (