fix loaders, allow larger websites

apps/cf-ai-backend/src/index.ts

@@ -69,21 +69,32 @@ app.post("/api/add", zValidator("json", vectorObj), async (c) => {
 	const newPageContent = body.pageContent?.replace(/<raw>.*?<\/raw>/g, "");
 
 	const chunks = chunkText(newPageContent, 1536);
-	if (chunks.length > 20) {
-		return c.json({
-			status: "error",
-			message:
-				"We are unable to process documents this size just yet, try something smaller",
+
+	const chunksOf20 = chunks.reduce((acc, chunk, index) => {
+		if (index % 20 === 0) {
+			acc.push([chunk]);
+		} else {
+			acc[acc.length - 1].push(chunk);
+		}
+		return acc;
+	}, [] as string[][]);
+
+	const accumChunkedInputs = [];
+
+	const promises = chunksOf20.map(async (chunkGroup) => {
+		const chunkedInput = await batchCreateChunksAndEmbeddings({
+			store,
+			body,
+			chunks: chunkGroup,
+			context: c,
 		});
-	}
-	const chunkedInput = await batchCreateChunksAndEmbeddings({
-		store,
-		body,
-		chunks: chunks,
-		context: c,
+
+		accumChunkedInputs.push(chunkedInput);
 	});
 
-	return c.json({ status: "ok", chunkedInput });
+	await Promise.all(promises);
+
+	return c.json({ status: "ok", chunkedInput: accumChunkedInputs });
 });
 
 app.post(

apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts

@@ -50,6 +50,7 @@ export class OpenAIEmbeddings {
 		const json = zodTypeExpected.safeParse(data);
 
 		if (!json.success) {
+			console.log(data);
 			throw new Error("Invalid response from OpenAI: " + json.error.message);
 		}
 

apps/web/app/(dash)/home/history.tsx

@@ -1,53 +1,50 @@
-import { getChatHistory } from '@repo/web/app/actions/fetchers';
-import { ArrowLongRightIcon } from '@heroicons/react/24/outline';
-import { Skeleton } from '@repo/ui/shadcn/skeleton';
-import Link from 'next/link';
-import { memo, useEffect, useState } from 'react';
-import { motion } from 'framer-motion';
-import { chatThreads } from '@/server/db/schema';
+import { getChatHistory } from "@repo/web/app/actions/fetchers";
+import { ArrowLongRightIcon } from "@heroicons/react/24/outline";
+import { Skeleton } from "@repo/ui/shadcn/skeleton";
+import Link from "next/link";
+import { memo, useEffect, useState } from "react";
+import { motion } from "framer-motion";
+import { chatThreads } from "@/server/db/schema";
 
 const History = memo(() => {
-  const [chatThreads_, setChatThreads] = useState<
-    (typeof chatThreads.$inferSelect)[] | null
-  >(null);
+	const [chatThreads_, setChatThreads] = useState<
+		(typeof chatThreads.$inferSelect)[] | null
+	>(null);
 
-  useEffect(() => {
-    (async () => {
-      const chatThreads = await getChatHistory();
-      if (!chatThreads.success || !chatThreads.data) {
-        console.error(chatThreads.error);
-        return;
-      }
-      setChatThreads(chatThreads.data.reverse().slice(0, 3));
-    })();
-  }, []);
+	useEffect(() => {
+		(async () => {
+			const chatThreads = await getChatHistory();
+			if (!chatThreads.success || !chatThreads.data) {
+				console.error(chatThreads.error);
+				return;
+			}
+			setChatThreads(chatThreads.data.reverse().slice(0, 3));
+		})();
+	}, []);
 
-  if (!chatThreads) {
-    return (
-      <>
-        <Skeleton className="w-[80%] h-4 bg-[#3b444b] "></Skeleton>
-        <Skeleton className="w-[40%] h-4 bg-[#3b444b] "></Skeleton>
-        <Skeleton className="w-[60%] h-4 bg-[#3b444b] "></Skeleton>
-      </>
-    );
-  }
-
-  return (
-    <ul className="text-base list-none space-y-3 text-[#b9b9b9] mt-8">
-      {chatThreads_?.map((thread) => (
-        <motion.li
-          initial={{ opacity: 0, filter: 'blur(1px)' }}
-          animate={{ opacity: 1, filter: 'blur(0px)' }}
-          className="flex items-center gap-2 truncate"
-        >
-          <ArrowLongRightIcon className="h-5" />{' '}
-          <Link prefetch={false} href={`/chat/${thread.id}`}>
-            {thread.firstMessage}
-          </Link>
-        </motion.li>
-      ))}
-    </ul>
-  );
+	return (
+		<ul className="text-base list-none space-y-3 text-[#b9b9b9] mt-8">
+			{!chatThreads_ && (
+				<>
+					<Skeleton className="w-[80%] h-4 bg-[#3b444b] "></Skeleton>
+					<Skeleton className="w-[40%] h-4 bg-[#3b444b] "></Skeleton>
+					<Skeleton className="w-[60%] h-4 bg-[#3b444b] "></Skeleton>
+				</>
+			)}
+			{chatThreads_?.map((thread) => (
+				<motion.li
+					initial={{ opacity: 0, filter: "blur(1px)" }}
+					animate={{ opacity: 1, filter: "blur(0px)" }}
+					className="flex items-center gap-2 truncate"
+				>
+					<ArrowLongRightIcon className="h-5" />{" "}
+					<Link prefetch={false} href={`/chat/${thread.id}`}>
+						{thread.firstMessage}
+					</Link>
+				</motion.li>
+			))}
+		</ul>
+	);
 });
 
 export default History;

apps/web/app/(dash)/home/page.tsx

@@ -91,7 +91,7 @@ function Page({ searchParams }: { searchParams: Record<string, string> }) {
 					...slap,
 					transition: { ...slap.transition, delay: 0.2 },
 				}}
-				className="text-center mx-auto bg-[linear-gradient(180deg,_#FFF_0%,_rgba(255,_255,_255,_0.00)_202.08%)]  bg-clip-text text-4xl tracking-tighter   text-transparent md:text-5xl"
+				className="text-center mx-auto bg-[linear-gradient(180deg,_#FFF_0%,_rgba(255,_255,_255,_0.00)_202.08%)]  bg-clip-text text-4xl tracking-tighter   text-transparent md:text-5xl pb-2"
 			>
 				<span>Ask your</span>{" "}
 				<span className="inline-flex items-center gap-2 bg-gradient-to-r to-blue-300 from-zinc-300 text-transparent bg-clip-text">
@@ -99,7 +99,7 @@ function Page({ searchParams }: { searchParams: Record<string, string> }) {
 				</span>
 			</motion.h1>
 
-			<div className="w-full pb-20 mt-12">
+			<div className="w-full pb-20 mt-10">
 				<QueryInput
 					initialQuery={query}
 					setQueryPresent={setQueryPresent}

apps/web/app/(dash)/home/queryinput.tsx

@@ -4,6 +4,8 @@ import React, { useState } from "react";
 import { FilterSpaces } from "./filterSpaces";
 import { ArrowRightIcon } from "@repo/ui/icons";
 import Image from "next/image";
+import { Switch } from "@repo/ui/shadcn/switch";
+import { Label } from "@repo/ui/shadcn/label";
 
 function QueryInput({
 	setQueryPresent,
@@ -22,6 +24,8 @@ function QueryInput({
 }) {
 	const [q, setQ] = useState(initialQuery || "");
 
+	const [proMode, setProMode] = useState(false);
+
 	const [selectedSpaces, setSelectedSpaces] = useState<
 		{ id: number; name: string }[]
 	>([]);
@@ -72,9 +76,22 @@ function QueryInput({
 							setSelectedSpaces={setSelectedSpaces}
 							initialSpaces={initialSpaces || []}
 						/>
-						<button type="submit" className="rounded-lg bg-[#369DFD1A] p-3">
-							<Image src={ArrowRightIcon} alt="Enter" />
-						</button>
+						<div className="flex items-center gap-4">
+							<div className="flex items-center gap-2">
+								<Label htmlFor="pro-mode" className="text-sm text-[#9B9B9B]">
+									Pro mode
+								</Label>
+								<Switch
+									value={proMode ? "on" : "off"}
+									onChange={() => setProMode((prev) => !prev)}
+									id="pro-mode"
+									about="Pro mode"
+								/>
+							</div>
+							<button type="submit" className="rounded-lg bg-[#369DFD1A] p-3">
+								<Image src={ArrowRightIcon} alt="Enter" />
+							</button>
+						</div>
 					</div>
 				</form>
 			</div>

apps/web/app/(dash)/menu.tsx

@@ -334,7 +334,7 @@ function Menu() {
 							/>
 							<p className="text-xs text-foreground-menu mt-2">Add</p>
 						</DialogTrigger>
-						{menuItems.map((item) => (
+						{menuItems.slice(1, 2).map((item) => (
 							<Link
 								aria-disabled={item.disabled}
 								href={item.disabled ? "#" : item.url}

apps/web/app/(landing)/Hero.tsx

@@ -77,18 +77,22 @@ function Hero() {
 
 			<AnimatedLogoCloud />
 			<div className="relative z-50">
-				<motion.img
+				<motion.iframe
 					{...{
 						...slap,
 						transition: { ...slap.transition, delay: 0.35 },
 					}}
-					src="/images/landing-hero.jpeg"
-					alt="Landing page background"
 					draggable="false"
-					className="z-40 md:mt-[-40px] hidden sm:block h-full max-w-[70vw] mx-auto md:w-full select-none px-5 !rounded-2xl"
+					className="z-40 relative md:mt-[-40px] hidden sm:block h-full max-w-[70vw] mx-auto md:w-full select-none px-5 !rounded-2xl"
 					style={{
 						borderRadius: "20px",
+						width: "100%",
+						height: "100%",
 					}}
+					src="https://customer-5xczlbkyq4f9ejha.cloudflarestream.com/111c4828c3587348bc703e67bfca9682/iframe?preload=true&poster=https%3A%2F%2Fcustomer-5xczlbkyq4f9ejha.cloudflarestream.com%2F111c4828c3587348bc703e67bfca9682%2Fthumbnails%2Fthumbnail.jpg%3Ftime%3D%26height%3D600"
+					loading="lazy"
+					allow="accelerometer; gyroscope; autoplay; encrypted-media; picture-in-picture;"
+					allowFullScreen={true}
 				/>
 				<div
 					className="absolute -z-10 left-0 top-[10%] h-32 w-[90%] overflow-x-hidden bg-[rgb(54,157,253)] bg-opacity-100  blur-[337.4px]"

packages/ui/shadcn/switch.tsx

@@ -11,7 +11,7 @@ const Switch = React.forwardRef<
 >(({ className, ...props }, ref) => (
 	<SwitchPrimitives.Root
 		className={cn(
-			"peer inline-flex h-6 w-11 shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=unchecked]:bg-input",
+			"peer inline-flex h-6 w-11 shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-white data-[state=unchecked]:bg-[#409EFD1A]",
 			className,
 		)}
 		{...props}