From 4daee14a8254b67152285a4a649f2c306c695a24 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sun, 16 Jun 2024 00:12:58 -0500 Subject: [PATCH] added way to save content and vectorize them. also refactored code and added a bypass to browser rendering --- apps/browser-rendering | 2 +- apps/web/app/(auth)/auth-buttons.tsx | 2 +- apps/web/app/(auth)/signin/page.tsx | 2 +- apps/web/app/(dash)/chat/chatWindow.tsx | 2 +- apps/web/app/(dash)/chat/page.tsx | 2 +- apps/web/app/(dash)/home/page.tsx | 2 +- apps/web/app/(dash)/layout.tsx | 2 +- apps/web/app/(dash)/memories/page.tsx | 121 ++++++---- apps/web/app/(landing)/page.tsx | 2 +- apps/web/app/actions/doers.ts | 220 +++++++++++++++++- apps/web/app/actions/fetchers.ts | 125 +++++++++- apps/web/app/actions/types.ts | 1 + apps/web/app/api/[...nextauth]/route.ts | 2 +- apps/web/app/api/chat/route.ts | 2 +- apps/web/app/api/ensureAuth.ts | 4 +- apps/web/app/api/getCount/route.ts | 8 +- apps/web/app/api/me/route.ts | 4 +- apps/web/app/api/spaces/route.ts | 4 +- apps/web/app/api/store/route.ts | 31 +-- apps/web/app/ref/page.tsx | 8 +- apps/web/cf-env.d.ts | 12 +- apps/web/env.d.ts | 8 - apps/web/{app/helpers => lib}/constants.ts | 6 + .../web/{app/helpers => }/lib/get-metadata.ts | 0 .../helpers => }/lib/get-theme-button.tsx | 0 .../{app/helpers => }/lib/handle-errors.ts | 0 .../web/{app/helpers => }/lib/searchParams.ts | 0 apps/web/{app/helpers => }/server/auth.ts | 0 apps/web/{app/helpers => }/server/db/index.ts | 0 .../web/{app/helpers => }/server/db/schema.ts | 8 +- package.json | 5 +- 31 files changed, 479 insertions(+), 106 deletions(-) rename apps/web/{app/helpers => lib}/constants.ts (86%) rename apps/web/{app/helpers => }/lib/get-metadata.ts (100%) rename apps/web/{app/helpers => }/lib/get-theme-button.tsx (100%) rename apps/web/{app/helpers => }/lib/handle-errors.ts (100%) rename apps/web/{app/helpers => }/lib/searchParams.ts (100%) rename apps/web/{app/helpers => }/server/auth.ts (100%) rename apps/web/{app/helpers => }/server/db/index.ts (100%) rename apps/web/{app/helpers => }/server/db/schema.ts (96%) diff --git a/apps/browser-rendering b/apps/browser-rendering index b37c9623..f1797d84 160000 --- a/apps/browser-rendering +++ b/apps/browser-rendering @@ -1 +1 @@ -Subproject commit b37c962365a36cf342a31a196f4908f4f1343553 +Subproject commit f1797d84ff322d98041c0909e65ef6db8f7aa2cd diff --git a/apps/web/app/(auth)/auth-buttons.tsx b/apps/web/app/(auth)/auth-buttons.tsx index 0e99213e..5b0ad06e 100644 --- a/apps/web/app/(auth)/auth-buttons.tsx +++ b/apps/web/app/(auth)/auth-buttons.tsx @@ -2,7 +2,7 @@ import { Button } from "@repo/ui/shadcn/button"; import React from "react"; -import { signIn } from "../helpers/server/auth"; +import { signIn } from "../../server/auth"; function SignIn() { return ( diff --git a/apps/web/app/(auth)/signin/page.tsx b/apps/web/app/(auth)/signin/page.tsx index ba84a94a..d7bad8da 100644 --- a/apps/web/app/(auth)/signin/page.tsx +++ b/apps/web/app/(auth)/signin/page.tsx @@ -1,7 +1,7 @@ import Image from "next/image"; import Link from "next/link"; import Logo from "@/public/logo.svg"; -import { signIn } from "@/app/helpers/server/auth"; +import { signIn } from "@/server/auth"; import { Google } from "@repo/ui/components/icons"; export const runtime = "edge"; diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index b631c835..d4c76469 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -19,7 +19,7 @@ import remarkMath from "remark-math"; import rehypeKatex from "rehype-katex"; import rehypeHighlight from "rehype-highlight"; import { code, p } from "./markdownRenderHelpers"; -import { codeLanguageSubset } from "@/app/helpers/constants"; +import { codeLanguageSubset } from "@/lib/constants"; function ChatWindow({ q, diff --git a/apps/web/app/(dash)/chat/page.tsx b/apps/web/app/(dash)/chat/page.tsx index fd4de826..73519851 100644 --- a/apps/web/app/(dash)/chat/page.tsx +++ b/apps/web/app/(dash)/chat/page.tsx @@ -1,5 +1,5 @@ import ChatWindow from "./chatWindow"; -import { chatSearchParamsCache } from "../../helpers/lib/searchParams"; +import { chatSearchParamsCache } from "../../../lib/searchParams"; // @ts-expect-error await import("katex/dist/katex.min.css"); diff --git a/apps/web/app/(dash)/home/page.tsx b/apps/web/app/(dash)/home/page.tsx index b4bafb38..55f2928e 100644 --- a/apps/web/app/(dash)/home/page.tsx +++ b/apps/web/app/(dash)/home/page.tsx @@ -2,7 +2,7 @@ import React from "react"; import Menu from "../menu"; import Header from "../header"; import QueryInput from "./queryinput"; -import { homeSearchParamsCache } from "@/app/helpers/lib/searchParams"; +import { homeSearchParamsCache } from "@/lib/searchParams"; import { getSpaces } from "@/app/actions/fetchers"; async function Page({ diff --git a/apps/web/app/(dash)/layout.tsx b/apps/web/app/(dash)/layout.tsx index b879a2f5..3ec8926e 100644 --- a/apps/web/app/(dash)/layout.tsx +++ b/apps/web/app/(dash)/layout.tsx @@ -1,7 +1,7 @@ import Header from "./header"; import Menu from "./menu"; import { redirect } from "next/navigation"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; import { Toaster } from "@repo/ui/shadcn/sonner"; async function Layout({ children }: { children: React.ReactNode }) { diff --git a/apps/web/app/(dash)/memories/page.tsx b/apps/web/app/(dash)/memories/page.tsx index bc2fcd53..ff746d1d 100644 --- a/apps/web/app/(dash)/memories/page.tsx +++ b/apps/web/app/(dash)/memories/page.tsx @@ -1,14 +1,31 @@ "use client"; +import { getAllUserMemoriesAndSpaces } from "@/app/actions/fetchers"; +import { Space } from "@/app/actions/types"; +import { Content } from "@/server/db/schema"; import { NextIcon, SearchIcon, UrlIcon } from "@repo/ui/icons"; import Image from "next/image"; -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; -function page() { - const [filter, setFilter] = useState("All") - const setFilterfn = (i:string) => setFilter(i) +function Page() { + const [filter, setFilter] = useState("All"); + const setFilterfn = (i: string) => setFilter(i); + + const [search, setSearch] = useState(""); + + const [memoriesAndSpaces, setMemoriesAndSpaces] = useState<{ + memories: Content[]; + spaces: Space[]; + }>({ memories: [], spaces: [] }); + + useEffect(() => { + (async () => { + const { success, data } = await getAllUserMemoriesAndSpaces(); + if (!success ?? !data) return; + setMemoriesAndSpaces({ memories: data.memories, spaces: data.spaces }); + })(); + }, []); - const [search, setSearch] = useState("") return (

@@ -16,41 +33,50 @@ function page() {

-
- - Search icon -
- - +
+ + Search icon +
+
Spaces
- - - + {memoriesAndSpaces.spaces.map((space) => ( + + ))}
Pages
- - - + {memoriesAndSpaces.memories.map((memory) => ( + + ))}
); } -function TabComponent({title, description}: {title:string, description:string}){ +function TabComponent({ + title, + description, +}: { + title: string; + description: string; +}) { return (
- {title.slice(0,2).toUpperCase()} + {title.slice(0, 2).toUpperCase()}
@@ -58,37 +84,50 @@ function TabComponent({title, description}: {title:string, description:string}){
{description}
- Search icon + Search icon
- ) + ); } -function LinkComponent({title, url}: {title:string, url:string}){ +function LinkComponent({ title, url }: { title: string; url: string }) { return (
-
-
- Url icon +
+
+ Url icon +
+
+
+
{title}
+
{url}
-
-
{title}
-
{url}
-
-
- ) + ); } -const FilterMethods = ["All", "Spaces", "Pages", "Notes"] -function Filters({setFilter, filter}:{setFilter: (i:string)=> void, filter: string}){ +const FilterMethods = ["All", "Spaces", "Pages", "Notes"]; +function Filters({ + setFilter, + filter, +}: { + setFilter: (i: string) => void; + filter: string; +}) { return (
- {FilterMethods.map((i)=> { - return
setFilter(i)} className={`transition px-6 py-2 rounded-xl ${i === filter ? "bg-[#21303D] text-[#369DFD]" : "text-[#B3BCC5] bg-[#1F2428] hover:bg-[#1f262d] hover:text-[#76a3cc]"}`}>{i}
+ {FilterMethods.map((i) => { + return ( +
setFilter(i)} + className={`transition px-6 py-2 rounded-xl ${i === filter ? "bg-[#21303D] text-[#369DFD]" : "text-[#B3BCC5] bg-[#1F2428] hover:bg-[#1f262d] hover:text-[#76a3cc]"}`} + > + {i} +
+ ); })}
- ) + ); } -export default page; +export default Page; diff --git a/apps/web/app/(landing)/page.tsx b/apps/web/app/(landing)/page.tsx index 09f94d92..5f8b28b4 100644 --- a/apps/web/app/(landing)/page.tsx +++ b/apps/web/app/(landing)/page.tsx @@ -5,7 +5,7 @@ import Cta from "./Cta"; import { Toaster } from "@repo/ui/shadcn/toaster"; import Features from "./Features"; import Footer from "./footer"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; import { redirect } from "next/navigation"; export const runtime = "edge"; diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index c8a1f3b4..798d40fe 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -1,10 +1,15 @@ "use server"; import { revalidatePath } from "next/cache"; -import { db } from "../helpers/server/db"; -import { space } from "../helpers/server/db/schema"; +import { db } from "../../server/db"; +import { contentToSpace, space, storedContent } from "../../server/db/schema"; import { ServerActionReturnType } from "./types"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; +import { Tweet } from "react-tweet/api"; +import { getMetaData } from "@/lib/get-metadata"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { LIMITS } from "@/lib/constants"; +import { z } from "zod"; export const createSpace = async ( input: string | FormData, @@ -41,3 +46,212 @@ export const createSpace = async ( } } }; + +const typeDecider = (content: string) => { + // if the content is a URL, then it's a page. if its a URL with https://x.com/user/status/123, then it's a tweet. else, it's a note. + // do strict checking with regex + if (content.match(/https?:\/\/[\w\.]+\/[\w]+\/[\w]+\/[\d]+/)) { + return "tweet"; + } else if (content.match(/https?:\/\/[\w\.]+/)) { + return "page"; + } else { + return "note"; + } +}; + +export const limit = async (userId: string, type = "page") => { + const count = await db + .select({ + count: sql`count(*)`.mapWith(Number), + }) + .from(storedContent) + .where(and(eq(storedContent.userId, userId), eq(storedContent.type, type))); + + if (count[0]!.count > LIMITS[type as keyof typeof LIMITS]) { + return false; + } + + return true; +}; + +const getTweetData = async (tweetID: string) => { + const url = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetID}&lang=en&features=tfw_timeline_list%3A%3Btfw_follower_count_sunset%3Atrue%3Btfw_tweet_edit_backend%3Aon%3Btfw_refsrc_session%3Aon%3Btfw_fosnr_soft_interventions_enabled%3Aon%3Btfw_show_birdwatch_pivots_enabled%3Aon%3Btfw_show_business_verified_badge%3Aon%3Btfw_duplicate_scribes_to_settings%3Aon%3Btfw_use_profile_image_shape_enabled%3Aon%3Btfw_show_blue_verified_badge%3Aon%3Btfw_legacy_timeline_sunset%3Atrue%3Btfw_show_gov_verified_badge%3Aon%3Btfw_show_business_affiliate_badge%3Aon%3Btfw_tweet_edit_frontend%3Aon&token=4c2mmul6mnh`; + + const resp = await fetch(url, { + headers: { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + Accept: "application/json", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + Connection: "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Cache-Control": "max-age=0", + TE: "Trailers", + }, + }); + console.log(resp.status); + const data = (await resp.json()) as Tweet; + + return data; +}; + +export const createMemory = async (input: { + content: string; + spaces?: string[]; +}): ServerActionReturnType => { + const data = await auth(); + + if (!data || !data.user || !data.user.id) { + return { error: "Not authenticated", success: false }; + } + + const type = typeDecider(input.content); + + let pageContent = input.content; + let metadata: Awaited>; + + if (!(await limit(data.user.id, type))) { + return { + success: false, + data: 0, + error: `You have exceeded the limit of ${LIMITS[type as keyof typeof LIMITS]} ${type}s.`, + }; + } + + if (type === "page") { + const response = await fetch("https://md.dhr.wtf/?url=" + input.content, { + headers: { + Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + }, + }); + pageContent = await response.text(); + metadata = await getMetaData(pageContent); + } else if (type === "tweet") { + const tweet = await getTweetData(input.content.split("/").pop() as string); + pageContent = JSON.stringify(tweet); + metadata = { + baseUrl: input.content, + description: tweet.text, + image: tweet.user.profile_image_url_https, + title: `Tweet by ${tweet.user.name}`, + }; + } else if (type === "note") { + pageContent = input.content; + const noteId = new Date().getTime(); + metadata = { + baseUrl: `https://supermemory.ai/note/${noteId}`, + description: `Note created at ${new Date().toLocaleString()}`, + image: "https://supermemory.ai/logo.png", + title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, + }; + } else { + return { + success: false, + data: 0, + error: "Invalid type", + }; + } + + let storeToSpaces = input.spaces; + + if (!storeToSpaces) { + storeToSpaces = []; + } + + // Insert into database + const insertResponse = await db + .insert(storedContent) + .values({ + content: pageContent, + title: metadata.title, + description: metadata.description, + url: input.content, + baseUrl: metadata.baseUrl, + image: metadata.image, + savedAt: new Date(), + userId: data.user.id, + type, + }) + .returning({ id: storedContent.id }); + + const contentId = insertResponse[0]?.id; + if (!contentId) { + return { + success: false, + data: 0, + error: "Something went wrong while saving the document to the database", + }; + } + + if (storeToSpaces.length > 0) { + // Adding the many-to-many relationship between content and spaces + const spaceData = await db + .select() + .from(space) + .where( + and(inArray(space.name, storeToSpaces), eq(space.user, data.user.id)), + ) + .all(); + + await Promise.all( + spaceData.map(async (space) => { + await db + .insert(contentToSpace) + .values({ contentId: contentId, spaceId: space.id }); + }), + ); + } + + const vectorSaveResponse = await fetch( + `${process.env.BACKEND_BASE_URL}/api/add`, + { + method: "POST", + body: JSON.stringify({ + pageContent, + title: metadata.title, + description: metadata.description, + url: metadata.baseUrl, + // TODO: now, in the vector store, we are only saving the first space. We need to save all spaces. + space: storeToSpaces[0], + user: data.user.id, + }), + }, + ); + + if (!vectorSaveResponse.ok) { + const errorData = await vectorSaveResponse.text(); + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${errorData}`, + }; + } + + try { + const response = await vectorSaveResponse.json(); + + const expectedResponse = z.object({ status: z.literal("ok") }); + + const parsedResponse = expectedResponse.safeParse(response); + + if (!parsedResponse.success) { + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${parsedResponse.error.message}`, + }; + } + + return { + success: true, + data: 1, + }; + } catch (e) { + return { + success: false, + data: 0, + error: `Failed to save to vector store. Backend returned error: ${e}`, + }; + } +}; diff --git a/apps/web/app/actions/fetchers.ts b/apps/web/app/actions/fetchers.ts index 9c2527f0..dc71252e 100644 --- a/apps/web/app/actions/fetchers.ts +++ b/apps/web/app/actions/fetchers.ts @@ -1,10 +1,15 @@ "use server"; -import { eq } from "drizzle-orm"; -import { db } from "../helpers/server/db"; -import { users } from "../helpers/server/db/schema"; +import { eq, inArray, not, sql } from "drizzle-orm"; +import { db } from "../../server/db"; +import { + Content, + contentToSpace, + storedContent, + users, +} from "../../server/db/schema"; import { ServerActionReturnType, Space } from "./types"; -import { auth } from "../helpers/server/auth"; +import { auth } from "../../server/auth"; export const getSpaces = async (): ServerActionReturnType => { const data = await auth(); @@ -23,3 +28,115 @@ export const getSpaces = async (): ServerActionReturnType => { return { success: true, data: spacesWithoutUser }; }; + +export const getAllMemories = async ( + freeMemoriesOnly: boolean = false, +): ServerActionReturnType => { + const data = await auth(); + + if (!data || !data.user) { + return { error: "Not authenticated", success: false }; + } + + if (!freeMemoriesOnly) { + // Returns all memories, no matter the space. + const memories = await db.query.storedContent.findMany({ + where: eq(users, data.user.id), + }); + + return { success: true, data: memories }; + } + + // This only returns memories that are not a part of any space. + // This is useful for home page where we want to show a list of spaces and memories. + const contentNotInAnySpace = await db + .select() + .from(storedContent) + .where( + not( + eq( + storedContent.id, + db + .select({ contentId: contentToSpace.contentId }) + .from(contentToSpace), + ), + ), + ) + .execute(); + + return { success: true, data: contentNotInAnySpace }; +}; + +export const getAllUserMemoriesAndSpaces = async (): ServerActionReturnType<{ + spaces: Space[]; + memories: Content[]; +}> => { + const data = await auth(); + + if (!data || !data.user) { + return { error: "Not authenticated", success: false }; + } + + const spaces = await db.query.space.findMany({ + where: eq(users, data.user.id), + }); + + const spacesWithoutUser = spaces.map((space) => { + return { ...space, user: undefined }; + }); + + // const contentCountBySpace = await db + // .select({ + // spaceId: contentToSpace.spaceId, + // count: sql`count(*)`.mapWith(Number), + // }) + // .from(contentToSpace) + // .where( + // inArray( + // contentToSpace.spaceId, + // spacesWithoutUser.map((space) => space.id), + // ), + // ) + // .groupBy(contentToSpace.spaceId) + // .execute(); + + // console.log(contentCountBySpace); + + // get a count with space mappings like spaceID: count (number of memories in that space) + const contentCountBySpace = await db + .select({ + spaceId: contentToSpace.spaceId, + count: sql`count(*)`.mapWith(Number), + }) + .from(contentToSpace) + .where( + inArray( + contentToSpace.spaceId, + spacesWithoutUser.map((space) => space.id), + ), + ) + .groupBy(contentToSpace.spaceId) + .execute(); + + console.log(contentCountBySpace); + + const contentNotInAnySpace = await db + .select() + .from(storedContent) + .where( + not( + eq( + storedContent.id, + db + .select({ contentId: contentToSpace.contentId }) + .from(contentToSpace), + ), + ), + ) + .execute(); + + return { + success: true, + data: { spaces: spacesWithoutUser, memories: contentNotInAnySpace }, + }; +}; diff --git a/apps/web/app/actions/types.ts b/apps/web/app/actions/types.ts index fbf669e2..5c5afc5c 100644 --- a/apps/web/app/actions/types.ts +++ b/apps/web/app/actions/types.ts @@ -1,6 +1,7 @@ export type Space = { id: number; name: string; + numberOfMemories?: number; }; export type ServerActionReturnType = Promise<{ diff --git a/apps/web/app/api/[...nextauth]/route.ts b/apps/web/app/api/[...nextauth]/route.ts index 50807ab1..e19cc16e 100644 --- a/apps/web/app/api/[...nextauth]/route.ts +++ b/apps/web/app/api/[...nextauth]/route.ts @@ -1,2 +1,2 @@ -export { GET, POST } from "../../helpers/server/auth"; +export { GET, POST } from "../../../server/auth"; export const runtime = "edge"; diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index aba8784c..541ced34 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -54,7 +54,7 @@ export async function POST(req: NextRequest) { ); const resp = await fetch( - `https://new-cf-ai-backend.dhravya.workers.dev/api/chat?query=${query}&user=${session.user.email}&sourcesOnly=${sourcesOnly}&spaces=${spaces}`, + `${process.env.BACKEND_BASE_URL}/api/chat?query=${query}&user=${session.user.email}&sourcesOnly=${sourcesOnly}&spaces=${spaces}`, { headers: { Authorization: `Bearer ${process.env.BACKEND_SECURITY_KEY}`, diff --git a/apps/web/app/api/ensureAuth.ts b/apps/web/app/api/ensureAuth.ts index a1401a07..d2fbac0b 100644 --- a/apps/web/app/api/ensureAuth.ts +++ b/apps/web/app/api/ensureAuth.ts @@ -1,6 +1,6 @@ import { NextRequest } from "next/server"; -import { db } from "../helpers/server/db"; -import { sessions, users } from "../helpers/server/db/schema"; +import { db } from "../../server/db"; +import { sessions, users } from "../../server/db/schema"; import { eq } from "drizzle-orm"; export async function ensureAuth(req: NextRequest) { diff --git a/apps/web/app/api/getCount/route.ts b/apps/web/app/api/getCount/route.ts index f760c145..7cd2a2d3 100644 --- a/apps/web/app/api/getCount/route.ts +++ b/apps/web/app/api/getCount/route.ts @@ -1,6 +1,6 @@ -import { db } from "@/app/helpers/server/db"; +import { db } from "@/server/db"; import { and, eq, ne, sql } from "drizzle-orm"; -import { sessions, storedContent, users } from "@/app/helpers/server/db/schema"; +import { sessions, storedContent, users } from "@/server/db/schema"; import { type NextRequest, NextResponse } from "next/server"; import { ensureAuth } from "../ensureAuth"; @@ -20,7 +20,7 @@ export async function GET(req: NextRequest) { .from(storedContent) .where( and( - eq(storedContent.user, session.user.id), + eq(storedContent.userId, session.user.id), eq(storedContent.type, "twitter-bookmark"), ), ); @@ -32,7 +32,7 @@ export async function GET(req: NextRequest) { .from(storedContent) .where( and( - eq(storedContent.user, session.user.id), + eq(storedContent.userId, session.user.id), ne(storedContent.type, "twitter-bookmark"), ), ); diff --git a/apps/web/app/api/me/route.ts b/apps/web/app/api/me/route.ts index 20b6aece..621dcbfe 100644 --- a/apps/web/app/api/me/route.ts +++ b/apps/web/app/api/me/route.ts @@ -1,6 +1,6 @@ -import { db } from "@/app/helpers/server/db"; +import { db } from "@/server/db"; import { eq } from "drizzle-orm"; -import { sessions, users } from "@/app/helpers/server/db/schema"; +import { sessions, users } from "@/server/db/schema"; import { type NextRequest, NextResponse } from "next/server"; export const runtime = "edge"; diff --git a/apps/web/app/api/spaces/route.ts b/apps/web/app/api/spaces/route.ts index c46b02fc..cbed547d 100644 --- a/apps/web/app/api/spaces/route.ts +++ b/apps/web/app/api/spaces/route.ts @@ -1,5 +1,5 @@ -import { db } from "@/app/helpers/server/db"; -import { sessions, space, users } from "@/app/helpers/server/db/schema"; +import { db } from "@/server/db"; +import { sessions, space, users } from "@/server/db/schema"; import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { ensureAuth } from "../ensureAuth"; diff --git a/apps/web/app/api/store/route.ts b/apps/web/app/api/store/route.ts index f96f90cf..cb10db24 100644 --- a/apps/web/app/api/store/route.ts +++ b/apps/web/app/api/store/route.ts @@ -1,4 +1,4 @@ -import { db } from "@/app/helpers/server/db"; +import { db } from "@/server/db"; import { and, eq, sql, inArray } from "drizzle-orm"; import { contentToSpace, @@ -6,10 +6,12 @@ import { storedContent, users, space, -} from "@/app/helpers/server/db/schema"; +} from "@/server/db/schema"; import { type NextRequest, NextResponse } from "next/server"; -import { getMetaData } from "@/app/helpers/lib/get-metadata"; +import { getMetaData } from "@/lib/get-metadata"; import { ensureAuth } from "../ensureAuth"; +import { limit } from "@/app/actions/doers"; +import { LIMITS } from "@/lib/constants"; export const runtime = "edge"; @@ -33,22 +35,13 @@ export async function POST(req: NextRequest) { storeToSpaces = []; } - const count = await db - .select({ - count: sql`count(*)`.mapWith(Number), - }) - .from(storedContent) - .where( - and( - eq(storedContent.user, session.user.id), - eq(storedContent.type, "page"), - ), - ); - - if (count[0]!.count > 100) { + if (!(await limit(session.user.id))) { return NextResponse.json( - { message: "Error", error: "Limit exceeded" }, - { status: 499 }, + { + message: "Error: Ratelimit exceeded", + error: `You have exceeded the limit of ${LIMITS["page"]} pages.`, + }, + { status: 429 }, ); } @@ -62,7 +55,7 @@ export async function POST(req: NextRequest) { baseUrl: metadata.baseUrl, image: metadata.image, savedAt: new Date(), - user: session.user.id, + userId: session.user.id, }) .returning({ id: storedContent.id }); diff --git a/apps/web/app/ref/page.tsx b/apps/web/app/ref/page.tsx index 9ace733a..b51a16bb 100644 --- a/apps/web/app/ref/page.tsx +++ b/apps/web/app/ref/page.tsx @@ -1,9 +1,9 @@ import { Button } from "@repo/ui/shadcn/button"; -import { auth, signIn, signOut } from "../helpers/server/auth"; -import { db } from "../helpers/server/db"; +import { auth, signIn, signOut } from "../../server/auth"; +import { db } from "../../server/db"; import { sql } from "drizzle-orm"; -import { users } from "../helpers/server/db/schema"; -import { getThemeToggler } from "../helpers/lib/get-theme-button"; +import { users } from "../../server/db/schema"; +import { getThemeToggler } from "../../lib/get-theme-button"; export const runtime = "edge"; diff --git a/apps/web/cf-env.d.ts b/apps/web/cf-env.d.ts index 98303f35..e98c36cf 100644 --- a/apps/web/cf-env.d.ts +++ b/apps/web/cf-env.d.ts @@ -1,6 +1,16 @@ declare global { namespace NodeJS { - interface ProcessEnv extends CloudflareEnv {} + interface ProcessEnv extends CloudflareEnv { + GOOGLE_CLIENT_ID: string; + GOOGLE_CLIENT_SECRET: string; + AUTH_SECRET: string; + R2_ENDPOINT: string; + R2_ACCESS_ID: string; + R2_SECRET_KEY: string; + R2_BUCKET_NAME: string; + BACKEND_SECURITY_KEY: string; + BACKEND_BASE_URL: string; + } } } diff --git a/apps/web/env.d.ts b/apps/web/env.d.ts index 2755280c..4f11ba55 100644 --- a/apps/web/env.d.ts +++ b/apps/web/env.d.ts @@ -2,14 +2,6 @@ // by running `wrangler types --env-interface CloudflareEnv env.d.ts` interface CloudflareEnv { - GOOGLE_CLIENT_ID: string; - GOOGLE_CLIENT_SECRET: string; - AUTH_SECRET: string; - R2_ENDPOINT: string; - R2_ACCESS_ID: string; - R2_SECRET_KEY: string; - R2_BUCKET_NAME: string; - BACKEND_SECURITY_KEY: string; STORAGE: R2Bucket; DATABASE: D1Database; } diff --git a/apps/web/app/helpers/constants.ts b/apps/web/lib/constants.ts similarity index 86% rename from apps/web/app/helpers/constants.ts rename to apps/web/lib/constants.ts index c3fc640a..7a9485cf 100644 --- a/apps/web/app/helpers/constants.ts +++ b/apps/web/lib/constants.ts @@ -1,3 +1,9 @@ +export const LIMITS = { + page: 100, + tweet: 1000, + note: 1000, +}; + export const codeLanguageSubset = [ "python", "javascript", diff --git a/apps/web/app/helpers/lib/get-metadata.ts b/apps/web/lib/get-metadata.ts similarity index 100% rename from apps/web/app/helpers/lib/get-metadata.ts rename to apps/web/lib/get-metadata.ts diff --git a/apps/web/app/helpers/lib/get-theme-button.tsx b/apps/web/lib/get-theme-button.tsx similarity index 100% rename from apps/web/app/helpers/lib/get-theme-button.tsx rename to apps/web/lib/get-theme-button.tsx diff --git a/apps/web/app/helpers/lib/handle-errors.ts b/apps/web/lib/handle-errors.ts similarity index 100% rename from apps/web/app/helpers/lib/handle-errors.ts rename to apps/web/lib/handle-errors.ts diff --git a/apps/web/app/helpers/lib/searchParams.ts b/apps/web/lib/searchParams.ts similarity index 100% rename from apps/web/app/helpers/lib/searchParams.ts rename to apps/web/lib/searchParams.ts diff --git a/apps/web/app/helpers/server/auth.ts b/apps/web/server/auth.ts similarity index 100% rename from apps/web/app/helpers/server/auth.ts rename to apps/web/server/auth.ts diff --git a/apps/web/app/helpers/server/db/index.ts b/apps/web/server/db/index.ts similarity index 100% rename from apps/web/app/helpers/server/db/index.ts rename to apps/web/server/db/index.ts diff --git a/apps/web/app/helpers/server/db/schema.ts b/apps/web/server/db/schema.ts similarity index 96% rename from apps/web/app/helpers/server/db/schema.ts rename to apps/web/server/db/schema.ts index e3e789c6..1ff23c82 100644 --- a/apps/web/app/helpers/server/db/schema.ts +++ b/apps/web/server/db/schema.ts @@ -103,11 +103,9 @@ export const storedContent = createTable( savedAt: int("savedAt", { mode: "timestamp" }).notNull(), baseUrl: text("baseUrl", { length: 255 }), ogImage: text("ogImage", { length: 255 }), - type: text("type", { enum: ["note", "page", "twitter-bookmark"] }).default( - "page", - ), + type: text("type").default("page"), image: text("image", { length: 255 }), - userId: int("user").references(() => users.id, { + userId: text("user").references(() => users.id, { onDelete: "cascade", }), }, @@ -119,6 +117,8 @@ export const storedContent = createTable( }), ); +export type Content = typeof storedContent.$inferSelect; + export const contentToSpace = createTable( "contentToSpace", { diff --git a/package.json b/package.json index 11e3ba06..dd384010 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "devDependencies": { "@clack/prompts": "^0.7.0", "@cloudflare/next-on-pages": "1", - "@cloudflare/workers-types": "^4.20240512.0", + "@cloudflare/workers-types": "^4.20240614.0", "@repo/eslint-config": "*", "@repo/tailwind-config": "*", "@repo/typescript-config": "*", @@ -45,7 +45,7 @@ "@auth/drizzle-adapter": "^1.1.0", "@aws-sdk/client-s3": "^3.577.0", "@aws-sdk/s3-request-presigner": "^3.577.0", - "@cloudflare/puppeteer": "^0.0.8", + "@cloudflare/puppeteer": "^0.0.11", "@headlessui/react": "^2.0.4", "@hono/swagger-ui": "^0.2.2", "@hookform/resolvers": "^3.4.2", @@ -78,6 +78,7 @@ "react-dropzone": "^14.2.3", "react-hook-form": "^7.51.5", "react-markdown": "^9.0.1", + "react-tweet": "^3.2.1", "rehype-highlight": "^7.0.0", "rehype-katex": "^7.0.0", "remark-gfm": "^4.0.0",