Supermemory v2 Release 🚀

.cursorrules

@@ -0,0 +1 @@
+i am building an app called "supermemory"

.eslintrc.js

@@ -1,13 +0,0 @@
-// This configuration only applies to the package manager root.
-/** @type {import("eslint").Linter.Config} */
-module.exports = {
-	ignorePatterns: ["apps/**", "packages/**"],
-	extends: ["@repo/eslint-config/library.js"],
-	parser: "@typescript-eslint/parser",
-	parserOptions: {
-		project: true,
-	},
-	rules: {
-		"@typescript-eslint/consistent-type-imports": "off",
-	},
-};

.gitattributes

@@ -1 +0,0 @@
-*.lockb binary diff=lockb

.github/actions/buildextension/action.yml

@@ -1,29 +0,0 @@
-name: "Build Chrome extension"
-description: "Builds the chrome extension into a ZIP"
-runs:
-  using: "composite"
-  steps:
-    - name: Setup Bun
-      uses: oven-sh/setup-bun@v1
-      with:
-        bun-version: latest
-
-    - name: Install packages
-      run: bun i
-      shell: bash
-
-    - name: Build extension
-      run: bun run build
-      working-directory: apps/extension
-      shell: bash
-
-    - name: Pack extension
-      working-directory: apps/extension/dist
-      run: zip -r extension-${{ github.sha }}.zip *
-      shell: bash
-
-    - name: Upload extension archive
-      uses: actions/upload-artifact@v3
-      with:
-        name: extension-${{ github.sha }}
-        path: apps/extension/dist/extension-${{ github.sha }}.zip

.github/workflows/release.yml

@@ -1,30 +0,0 @@
-name: Upload to Web Store
-on:
-  push:
-    tags:
-      - "*"
-jobs:
-  build-extension:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/buildextension
-
-  upload-extension:
-    runs-on: ubuntu-latest
-    needs: build-extension
-    steps:
-      - name: Download extension archive
-        uses: actions/download-artifact@v3
-        with:
-          name: extension-${{ github.sha }}
-
-      - name: Upload to webstore
-        uses: ExtensionNinja/extension-publish@main
-        with:
-          action: upload
-          extensionID: afpgkkipfdpeaflnpoaffkcankadgjfc
-          clientID: ${{ secrets.GOOGLE_CLIENT_ID }}
-          clientSecret: ${{ secrets.GOOGLE_CLIENT_SECRET }}
-          clientRefreshToken: ${{ secrets.GOOGLE_REFRESH_TOKEN }}
-          extensionFile: apps/extension/dist/extension-${{ github.sha }}.zip

.gitignore

@@ -1,13 +1,9 @@
-.vars
-.*.vars
-.wrangler
-.million
-yarn.lock
-package-lock.json
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+bun.lockb
 
 # Dependencies
-node_modules
-.pnp
+node_modules/
+/.pnp
 .pnp.js
 
 # Local env files
@@ -16,9 +12,10 @@ node_modules
 .env.development.local
 .env.test.local
 .env.production.local
+.env*.local
 
 # Testing
-coverage
+/coverage
 
 # Turbo
 .turbo
@@ -27,11 +24,10 @@ coverage
 .vercel
 
 # Build Outputs
-.next/
+/.next
 out/
 build
-dist
-
+/dist
 
 # Debug
 npm-debug.log*
@@ -41,3 +37,28 @@ yarn-error.log*
 # Misc
 .DS_Store
 *.pem
+
+# IDE specific files
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Sensitive data
+*.key
+**/credentials.*
+**/secrets.*
+config.private.*
+
+# Cache
+.cache/
+.npm
+.eslintcache
+
+# Local database files
+*.sqlite
+*.db
+
+# Personal notes/todos
+TODO.md
+NOTES.md

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "apps/browser-rendering"]
-	path = apps/browser-rendering
-	url = https://github.com/dhravya/markdowner

.husky/_/pre-commit

@@ -1,3 +0,0 @@
-#!/bin/sh
-. "$(dirname "$0")/husky.sh"
-pnpm dlx lint-staged

.npmrc

@@ -1,4 +1,2 @@
-registry=https://registry.npmjs.org/
-link-workspace-packages=true
-@tiptap-pro:registry=https://registry.tiptap.dev/
-//registry.tiptap.dev/:_authToken=${TIPTAP_TOKEN}
+save-exact=true
+engine-strict=true

.prettierignore

@@ -1 +0,0 @@
-pnpm-lock.yaml

.prettierrc

@@ -1,5 +0,0 @@
-{
-	"tabWidth": 2,
-	"trailingComma": "all",
-	"useTabs": true
-}

.vscode/settings.json

@@ -1,7 +1,17 @@
 {
-	"css.validate": false,
-	"editor.quickSuggestions": {
-		"strings": true
+	"editor.formatOnSave": true,
+	"editor.defaultFormatter": "esbenp.prettier-vscode",
+	"editor.codeActionsOnSave": {
+		"source.fixAll.eslint": "always"
 	},
-	"typescript.tsdk": "node_modules/typescript/lib"
+	"sqltools.connections": [
+		{
+			"previewLimit": 50,
+			"server": "localhost",
+			"driver": "PostgreSQL",
+			"name": "SupermemoryDB",
+			"connectString": "postgresql://dhravya:DPOSTGRES99%40c22%23hLab27postgres@168.119.111.189/supermemory"
+		}
+	],
+	"millionLint.disableOutdatedVersionMessage": true
 }

Dockerfile

@@ -0,0 +1,37 @@
+FROM postgres:17
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    curl \
+    pkg-config \
+    libssl-dev \
+    libclang-dev \
+    llvm-dev \
+    postgresql-server-dev-all \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install pgvector
+RUN git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git \
+    && cd pgvector \
+    && make \
+    && make install
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install and initialize pgrx
+RUN cargo install cargo-pgrx --version 0.12.5 --locked && \
+    cargo pgrx init --pg17 pg_config
+
+# Clone and install pgvectorscale
+RUN cd /tmp && \
+    git clone --branch 0.5.0 https://github.com/timescale/pgvectorscale && \
+    cd pgvectorscale/pgvectorscale && \
+    cargo pgrx install --release
+
+# Create initialization script to enable both extensions
+RUN echo 'CREATE EXTENSION IF NOT EXISTS vector;' > /docker-entrypoint-initdb.d/01-init-vector.sql && \
+    echo 'CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;' > /docker-entrypoint-initdb.d/02-init-vectorscale.sql

LICENSE

@@ -1,21 +1,441 @@
-MIT License
+Supermemory
+Copyright (c) 2025 Dhravya Shah
+This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.
 
-Copyright (c) 2024 Dhravya Shah
+Attribution-NonCommercial-ShareAlike 4.0 International
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+=======================================================================
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
+Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-ShareAlike 4.0 International Public License
+("Public License"). To the extent this Public License may be
+interpreted as a contract, You are granted the Licensed Rights in
+consideration of Your acceptance of these terms and conditions, and the
+Licensor grants You such rights in consideration of benefits the
+Licensor receives from making the Licensed Material available under
+these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. BY-NC-SA Compatible License means a license listed at
+     creativecommons.org/compatiblelicenses, approved by Creative
+     Commons as essentially the equivalent of this Public License.
+
+  d. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  e. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  f. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  g. License Elements means the license attributes listed in the name
+     of a Creative Commons Public License. The License Elements of this
+     Public License are Attribution, NonCommercial, and ShareAlike.
+
+  h. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  i. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  j. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  k. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  l. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  m. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  n. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. Additional offer from the Licensor -- Adapted Material.
+               Every recipient of Adapted Material from You
+               automatically receives an offer from the Licensor to
+               exercise the Licensed Rights in the Adapted Material
+               under the conditions of the Adapter's License You apply.
+
+            c. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+  b. ShareAlike.
+
+     In addition to the conditions in Section 3(a), if You Share
+     Adapted Material You produce, the following conditions also apply.
+
+       1. The Adapter's License You apply must be a Creative Commons
+          license with the same License Elements, this version or
+          later, or a BY-NC-SA Compatible License.
+
+       2. You must include the text of, or the URI or hyperlink to, the
+          Adapter's License You apply. You may satisfy this condition
+          in any reasonable manner based on the medium, means, and
+          context in which You Share Adapted Material.
+
+       3. You may not offer or impose any additional or different terms
+          or conditions on, or apply any Effective Technological
+          Measures to, Adapted Material that restrict exercise of the
+          rights granted under the Adapter's License You apply.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material,
+     including for purposes of Section 3(b); and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

README.md

@@ -1,9 +1,7 @@
+
 ![og image](https://supermemory.ai/og-image.png)
 
-# SuperMemory
-
-Interested in helping build the best second brain for everyone? Join the discord https://discord.gg/2X2XsKz5AU. Contributions welcome.
-
+# Supermemory
 <div align="center">
   <a href="https://github.com/Dhravya/Supermemory/stargazers">
     <img src="https://img.shields.io/github/stars/Dhravya/Supermemory?style=flat-square&logo=github" alt="GitHub stars">
@@ -18,18 +16,29 @@ Interested in helping build the best second brain for everyone? Join the discord
     <img src="https://img.shields.io/chrome-web-store/v/afpgkkipfdpeaflnpoaffkcankadgjfc?style=flat-square&color=yellow" alt="Chrome Web Store">
   </a>
 </div>
+<br>
+<div align="center">
+<a href="https://www.producthunt.com/posts/supermemory?embed=true&utm_source=badge-top-post-badge&utm_medium=badge&utm_souce=badge-supermemory" target="_blank"><img src="https://api.producthunt.com/widgets/embed-image/v1/top-post-badge.svg?post_id=472686&theme=light&period=daily&t=1737414816950" alt="Supermemory - AI&#0032;second&#0032;brain&#0032;for&#0032;all&#0032;your&#0032;saved&#0032;stuff | Product Hunt" style="width: 250px; height: 54px;" width="250" height="54" /></a>
+</div>
+
+
 
 ## 👀 What is this?
 
-Build your own second brain with supermemory. It's a ChatGPT for your bookmarks. Import tweets or save websites and content using the [Chrome extension](https://chromewebstore.google.com/detail/supermemory/afpgkkipfdpeaflnpoaffkcankadgjfc?hl=en-GB&authuser=0)
+Supermemory exists to bring contextual knowledge to the age of LLMs. 
+The vision is to be the universal engine for memory
 
-Well, here's the thing - me and @yxshv save a _lot_ of content on the internet.
+LLMs are smart as fuck. but giving them the right context make them even more useful in most contexts.
 
-Twitter bookmarks, websites, snippets, etc.
+Think about it - there’s countless use-cases where LLMs by themselves don’t do much, but giving them the right data can make it a magical experience.
 
-But we never look back to it - to us, it's like throwing information in the void.
+Right data comes from everywhere. it’s literally with us. at all times. everything we see, hear, touch. is good data, USEFUL data. but then it all comes down to “how can i search for something within this pool of awesome stuff?”
 
-Supermemory fixes this.
+We make the pool, and give the search tools.
+
+That’s literally all Supermemory does. Every single project in our scope aids in doing one of the two things.
+
+Whether it’s organizing personal information, enhancing applications with contextual intelligence, or enabling companies to centralize and retrieve internal knowledge, Supermemory becomes the core infrastructure for transforming scattered data into actionable insights.
 
 ## Key Features
 
@@ -39,93 +48,96 @@ Supermemory fixes this.
 - 🐦 **Twitter Bookmarks**: Import and utilize your saved tweets.
 - 🔍 **Powerful Search**: Quickly find any saved information.
 - 💬 **Chat with Collections**: Interact with specific knowledge bases.
-- 🖼️ **Knowledge Canvas**: Organize information visually in a 2D canvas.
-- ✍️ **Writing Assistant**: Use a markdown editor with AI assistance for content creation.
+- 🖼️ **Memory refresh**: Contextually shows relevant items for re-learning.
+- ✍️ (soon) **Writing Assistant**: Use a markdown editor with AI assistance for content creation.
 - 🔒 **Privacy Focused**: Ensures data security and privacy.
 - 🏠 **Self Hostable**: Open source and easy to deploy locally.
-- 🔗 **Integrations**: Compatible with Telegram, Twitter, and more to come.
+- 🔗 **Integrations**: Compatible with Telegram, Twitter, Chrome bookmarks, Notion, and more to come.
+
+## Supermemory API
+Developers can make apps on top of supermemory using the [API](https://api.supermemory.ai). You can find the documentation here: https://docs.supermemory.ai
+
+YOU can build cool stuff with supermemory.
+1. Import tools 
+2. Chat with (insert whatever here) apps 
+3. Personalisation in existing apps 
+4. Content management systems
 
 ## How do I use this?
 
-Just go to [supermemory.ai](https://supermemory.ai) and sign in with your google account.
+Just go to [supermemory.com](https://supermemory.com) and sign in.
 
 To use the chrome extension,
 
-> [!WARNING]
-> You need to be signed in before installing the supermemory extension, or you may experience problems
-
 1. Download from [Chrome Web Store](https://chromewebstore.google.com/detail/supermemory/afpgkkipfdpeaflnpoaffkcankadgjfc?authuser=0&hl=en-GB)
-2. Now you can see on any page on bottom right (just click on it to save)
-   <img width="1058" alt="image" src="https://github.com/MaheshtheDev/supermemory/assets/38828053/f24b0988-cd2b-4216-a75b-e9ff0dbfaa6a">
+2. Pin the supermemory chrome extension and just click on it to save the website.
+   <img width="1058" alt="image" src="https://i.dhr.wtf/r/Clipboard_Jan_20,_2025_at_4.03 PM.png">
 
-### Import Twitter Bookmarks
+### Importing from integrations (Twitter, Chrome bookmarks, Notion, etc)
 
-1. Make sure you signed into supermemory and installed chrome extension
-2. Open Twitter/X, you will see the save icon as follows
-
-   <img width="480" alt="image" src="https://github.com/MaheshtheDev/supermemory/assets/38828053/2efb06a5-912a-48e7-ad1c-d527e7ffbc94">
-
-3. Click on save button and give it 10 - 20 secs, where supermemory extension will sync all your twitter bookmarks to supermemory.ai
-4. Voila! Now your second brain has all your twitter bookmarks.
-
-## 👨‍💻 The Stack
-
-<div align="center">
-<img src="https://tech-orbit.wontory.dev/api?title=SuperMemory&tech=React,Next.js,Tailwind%20CSS,shadcn/ui,Drizzle,Cloudflare,Cloudflare%20Pages,Cloudflare%20Workers&size=900&duration=20" alt="SuperMemory" width="400">
-</div>
+1. Open [Supermemory](https://supermemory.ai)
+2. Follow the steps to connect your integrations
+   <img width="480" alt="image" src="https://i.dhr.wtf/r/Clipboard_Jan_20,_2025_at_3.15 PM.png">
+3. Voila! Now your second brain has all your twitter bookmarks.
 
 #### Architecture:
 
-<img width="715" alt="image" src="https://github.com/user-attachments/assets/f2e38b28-8cfe-4331-9b84-a91902682ddd">
+<img width="715" alt="image" src="https://i.dhr.wtf/r/Clipboard_Jan_20,_2025_at_3.51 PM.png">
 
 
 Supermemory has three main modules, managed by [turborepo](https://turbo.build):
 
 #### `apps/web`: The main web UI.
 
-The database, auth etc logic is here
-
-![image](https://github.com/MaheshtheDev/supermemory/assets/38828053/0c44708d-600a-43a8-a641-835cb1f349fc)
+![image](https://i.dhr.wtf/r/Clipboard_Jan_20,_2025_at_3.19 PM.png)
 
 Built with:
 
-- [Nextjs 14](https://nextjs.org/)
-- [Next Auth](https://next-auth.js.org/)
+- [Remix](https://remix.run/)
+- [Hono](https://hono.dev/)
+- [authkit-remix-cloudflare by Supermemory](https://github.com/supermemory/authkit-remix-cloudflare)
 - [Drizzle ORM](https://drizzle.team/)
-- [Cloudflare D1 database](https://developers.cloudflare.com/d1/get-started/)
-- [Cloudflare ratelimiter](https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/)
 - [TailwindCSS](https://tailwindcss.com)
 - [shadcn-ui](https://ui.shadcn.com)
-- And some other amazing open source projects like [Novel](https://novel.sh) and [vaul](https://vaul.emilkowal.ski/)
+- And some other amazing open source projects like [Plate](https://platejs.org/) and [vaul](https://vaul.emilkowal.ski/)
 - Hosted on [Cloudflare Pages](https://pages.cloudflare.com/)
 
 #### `apps/extension`: Chrome extension
 
-The chrome extension is one of the most important part of the setup, but is not required.This is to easily add pages to your memory.
+The [chrome extension](https://supermemory.ai/extension) is one of the most important part of the setup, but is not required.This is to easily add pages to your memory.
 
-<img width="290" alt="image" src="https://github.com/MaheshtheDev/supermemory/assets/38828053/fa4993bb-c447-46a8-a301-9e0fa84d406a">
+<img width="290" alt="image" src="https://i.dhr.wtf/r/Clipboard_Jan_20,_2025_at_4.05 PM.png">
+
+> please rate the extension to improve the rating 🙏. 
 
 Built with:
 
-- [CRXJS](https://crxjs.dev/vite-plugin/getting-started/react/create-project)
-- [Vite](https://vitejs.dev/)
+- [Extension JS](https://extension.js.org)
 - [TailwindCSS](https://tailwindcss.com)
-- [shadcn-ui](https://ui.shadcn.com)
 - [React](https://react.dev/)
 
-#### `apps/cf-ai-backend`: This module handles the vector store and AI response generation
+#### `apps/backend`: This module handles the vector store and AI response generation
 
 This is where the magic happens!
 Built with:
 
 - [Cloudflare Workers](https://workers.cloudflare.com/)
-- [Cloudflare AI](https://ai.cloudflare.com)
-- [Cloudflare Vectorize](https://developers.cloudflare.com/vectorize/)
-- [Cloudflare Queues](https://developers.cloudflare.com/queues/)
-- [Cloudflare Browser Rendering](https://developers.cloudflare.com/browser-rendering/)
+- [Postgres + Pgvector with Pgvectorscale](https://github.com/timescale/pgvectorscale)
+- [Cloudflare Workflows](https://developers.cloudflare.com/queues/)
+- [R2 Object storage](https://developers.cloudflare.com/r2/)
+- [Markdowner by Supermemory](https://md.dhr.wtf)
 - [Cloudflare KV](https://developers.cloudflare.com/kv)
 - [mem0](https://app.mem0.ai)
 
+## Is this free?
+
+Yes, everything is free & open source.
+Supermemory is built by [me](https://dhravya.dev), a college student. My life situations make it very difficult and almost impossible to monetise the product.
+
+Any kind of sponsorships / support would mean a lot to me, and help me keep supermemory alive.
+
+You can sponsor on my Github sponsors page - https://github.com/sponsors/dhravya
+
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=Dhravya/supermemory&type=Date)](https://star-history.com/#Dhravya/supermemory&Date)
@@ -134,12 +146,21 @@ Built with:
 
 Supermemory is design to be set up easily locally and super duper easy to set up 💫
 
-Please see the [SETUP-GUIDE.md](SETUP-GUIDE.md) for setup instructions.
+Please see the [SELF-HOSTING-GUIDE.md](SELF-HOSTING-GUIDE.md) for setup instructions.
+
+## License
+
+Supermemory is licensed under the [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License](LICENSE).
+
+- You can use the code for personal projects, given appropriate attribution.
+- For non-commercial use, the code must be open source.
+- Please reach out to me if you want to use the code for commercial projects.
 
 ### Contributing
 
 Contributions are very welcome! A contribution can be as small as a ⭐ or even finding and creating issues.
 
+Thanks to all the awesome people who have contributed to supermemory.
 <a href="https://github.com/Dhravya/SuperMemory/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=Dhravya/SuperMemory" />
 </a>

SELF-HOSTING-GUIDE.md

@@ -0,0 +1,90 @@
+# Supermemory self-hosting guide
+
+## Local Setup
+
+### 1. Database Setup
+
+To spin up the database locally, use Docker Compose:
+
+```bash
+docker-compose up -d
+```
+
+This will start a PostgreSQL database with pgvector extension at `localhost:5432`.
+
+### 2. Database Migrations
+
+To generate a migration:
+
+```bash
+bun run generate-migration
+```
+
+To apply migrations:
+
+```bash
+bun run migrate:local
+```
+
+> Note: You MUST use the drizzle-orm functions exported from `packages/db` for interacting with the database. Not using them will cause type errors that are hard to debug.
+
+### 3. Environment Variables
+
+#### Backend (`apps/backend/.env` and `apps/backend/.dev.vars`):
+
+```env
+WORKOS_API_KEY=your_workos_api_key
+WORKOS_CLIENT_ID=your_workos_client_id 
+WORKOS_COOKIE_PASSWORD=your_cookie_password
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/supermemory"
+CONTENT_WORKFLOW=your_content_workflow
+GEMINI_API_KEY=your_gemini_api_key
+NODE_ENV=development
+OPEN_AI_API_KEY=your_openai_api_key
+BRAINTRUST_API_KEY=your_braintrust_api_key
+RESEND_API_KEY=your_resend_api_key
+TURNSTILE_SECRET_KEY=your_turnstile_secret_key
+```
+
+#### Web (`apps/web/.env` and `apps/web/.dev.vars`):
+
+```env
+WORKOS_CLIENT_ID=your_workos_client_id
+WORKOS_API_KEY=your_workos_api_key
+WORKOS_REDIRECT_URI="http://localhost:3000/callback"
+WORKOS_COOKIE_PASSWORD=your_cookie_password
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/supermemorydhravya"
+
+CLOUDFLARE_ACCOUNT_ID=your_cloudflare_account_id
+R2_ACCESS_KEY_ID=your_r2_access_key_id
+R2_SECRET_ACCESS_KEY=your_r2_secret_access_key
+
+BACKEND_URL=http://localhost:8787
+OPENAI_API_KEY=your_openai_api_key
+NOTION_CLIENT_ID=your_notion_client_id
+NOTION_CLIENT_SECRET=your_notion_client_secret
+
+NODE_ENV=development
+STRIPE_CHECKOUT_KEY=your_stripe_checkout_key
+STRIPE_WEBHOOK_SECRET=your_stripe_webhook_secret
+```
+
+You also need to update the Wrangler config for the web app and backend to your own account's resources on Cloudflare.
+
+### 4. Schema Changes
+
+To edit the database schema, modify the files in `packages/db/schema.ts`, and then repeat the steps in the [Database Migrations](#2-database-migrations) section.
+
+### 5. Running the Application
+
+1. Install dependencies:
+
+```bash
+bun install
+```
+
+2. Start the development servers:
+
+```bash
+bun run dev
+```

SETUP-GUIDE.md

@@ -1,137 +0,0 @@
-# Self Hosting Guide
-
-This guide will help you set up your own instance of Supermemory. This is necessary if you want to contribute to the project or if you want to self host the project. You can read more about the stack [here](https://github.com/supermemoryai/supermemory/?tab=readme-ov-file#-the-stack).
-
-## Prerequisites
-
-- [pnpm](https://pnpm.io/installation): pnpm is used as a package manager. You can enable pnpm by running `corepack enable pnpm` in your terminal.
-- [turbo](https://turbo.build/repo/docs/installing)
-- [wrangler](https://developers.cloudflare.com/workers/cli-wrangler/install-update)
-- [Cloudflare Workers](https://developers.cloudflare.com/workers/platform/pricing/): You also need to have a paid Workers plan to use the vectorize feature which is needed run the AI backend. It is currently $5/mo + usage costs.
-- [Cloudflare R2](https://developers.cloudflare.com/r2/): You need to enable R2 in the Cloudflare Dashboard for use in the web app.
-
-## Steps
-
-1. Clone the repo
-2. Run `pnpm install` in the root directory
-
-### web
-
-1. You need to create OAuth credentials for Google which is need for auth.js (nextauth). Visit https://developers.google.com/identity/protocols/oauth2 to learn more and https://console.cloud.google.com/apis/dashboard to create a new project and OAuth credentials. You need to set the redirect URL to `http://localhost:3000/api/auth/callback/google` for development. You can also set the redirect URL to your own domain if you are deploying the app.
-2. Create a `.dev.vars` file in `apps/web` with the following content:
-
-```bash
-GOOGLE_CLIENT_ID="" // required
-GOOGLE_CLIENT_SECRET="" // required
-NEXTAUTH_SECRET="" // generate by running `openssl rand -base64 32`
-DATABASE_URL='database.sqlite'
-NEXTAUTH_URL='http://localhost:3000'
-BACKEND_SECURITY_KEY="" // used to authenticate with the backend. generate a random string using `openssl rand -base64 32`
-BACKEND_BASE_URL="http://localhost:8686"
-```
-
-> [!NOTE]
-> The `BACKEND_SECURITY_KEY` should be the same as the `SECURITY_KEY` in the `.dev.vars` file in `apps/cf-ai-backend`.
-
-3. KV Namespaces
-
-```bash
-pnpx wrangler kv namespace create canvas-snaps
-```
-
-```bash
-pnpx wrangler kv namespace create recommendations
-```
-
-Do not change the binding value in the `wrangler.toml` but update the id for the namespaces with the values you get from the above commands.
-
-4. R2 Storage
-
-```bash
-pnpx wrangler r2 bucket create supermemory-r2
-```
-
-Update bucket_name in the `wrangler.toml` file in `apps/web` to `supermemory-r2`
-
-5. D1 Database
-
-```bash
-pnpx wrangler d1 create supermemory-db-prod
-```
-
-Update the database_name and database_id in `[[env.production.d1_databases]]` with the values you get from the above command.
-
-```bash
-pnpx wrangler d1 create supermemory-db-preview
-```
-
-Update the database_name and database_id in `[[d1_databases]]` and `[[env.preview.d1_databases]]` with the values you get from the above command.
-
-> [!NOTE]
-> please don't change the binding value even if wrangler cli suggests you to do so.
-
-```bash
-[[d1_databases]]
-binding = "DATABASE"
-database_name = "supermemory-db-preview"
-database_id = "YOUR_DB_ID"
-```
-
-Simply run this command in `apps/web`
-
-```bash
-pnpx wrangler d1 migrations apply supermemory-db-preview
-```
-
-If it runs, you can set up the cloud database as well by add the `--remote` flag,
-
-if you just want to contribute to frontend then just run `pnpm run dev` in the root of the project and done! (you won't be able to try ai stuff), otherwise continue...
-
-### cf-ai-backend
-
-1. You need to host your own worker for the `apps/cf-ai-backend` module.
-
-To do this, first edit the `.dev.vars` file in `apps/cf-ai-backend` with the following content:
-
-```bash
-SECURITY_KEY="veryrandomsecuritykey" // same as BACKEND_SECURITY_KEY in web
-// Why? to generate embeddings with 4000+ tokens
-OPENAI_API_KEY="sk-"
-```
-
-2. Run this command to initialise vector database
-   > Note: You need to use the workers paid plan to use vectorize for now.
-
-```bash
-pnpx wrangler vectorize create --dimensions=1536 supermemory --metric=cosine
-```
-
-Update the index_name for `[[vectorize]]` in `wrangler.toml` file in `apps/cf-ai-backend` with the `supermemory` or the name you used in the above command.
-
-3. Create KV namespaces for the `cf-ai-backend` module
-
-```bash
-pnpx wrangler kv namespace create prod
-```
-
-Update the id in `[[kv_namespaces]]` in the `wrangler.toml` file in `apps/cf-ai-backend` with the value you get from the above command.
-
-```bash
-pnpx wrangler kv namespace create preview
-```
-
-Update the preview_id in `[[kv_namespaces]]` in the `wrangler.toml` file in `apps/cf-ai-backend` with the value you get from the above command.
-
-## Local Development
-
-- Run `pnpm dev` in the root directory and Voila! You have your own supermemory instance running!
-
-> [!NOTE]
-> It sometimes takes multiple tries to successfully run the `pnpm dev` command. If you encounter any issues, try running the command again.
-
-## Deploying
-
-To deploy the web app, run `pnpm run deploy` in the `apps/web` directory.
-To deploy the cf-ai-backend module, run `wrangler publish` in the `apps/cf-ai-backend` directory.
-
-To get the extension running, you need to build it first. Run `pnpm build` in the `apps/extension` directory and then load the extension in chrome.

apps/backend/.gitignore

@@ -0,0 +1,35 @@
+# prod
+dist/
+.dev.vars
+*.vars
+
+# dev
+.yarn/
+!.yarn/releases
+.vscode/*
+!.vscode/launch.json
+!.vscode/*.code-snippets
+.idea/workspace.xml
+.idea/usage.statistics.xml
+.idea/shelf
+
+# deps
+node_modules/
+.wrangler
+
+# env
+.env
+.env.production
+.dev.vars
+
+# logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+# misc
+.DS_Store

apps/backend/README.md

@@ -0,0 +1,8 @@
+```
+npm install
+npm run dev
+```
+
+```
+npm run deploy
+```

apps/backend/drizzle.config.prod.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "drizzle-kit";
+
+export default defineConfig({
+	dialect: "postgresql",
+	schema: "../../packages/db",
+	out: "./drizzle",
+	dbCredentials: {
+		url: process.env.PROD_DATABASE_URL!,
+	},
+});

apps/backend/drizzle.config.ts

@@ -0,0 +1,18 @@
+import { config } from "dotenv";
+import { defineConfig } from "drizzle-kit";
+import process from "process";
+
+config();
+
+if (process.env.NODE_ENV !== "production" && !process.env.DATABASE_URL) {
+	throw new Error("DATABASE_URL is not set");
+}
+
+export default defineConfig({
+	dialect: "postgresql",
+	schema: "../../packages/db",
+	out: "./drizzle",
+	dbCredentials: {
+		url: process.env.DATABASE_URL!,
+	},
+});

apps/backend/drizzle/0000_odd_impossible_man.sql

@@ -0,0 +1,189 @@
+CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;
+
+CREATE TABLE IF NOT EXISTS "chat_threads" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"uuid" varchar(36) NOT NULL,
+	"firstMessage" text NOT NULL,
+	"user_id" integer NOT NULL,
+	"messages" jsonb NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "chat_threads_uuid_unique" UNIQUE("uuid")
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "chunks" (
+	"id" serial PRIMARY KEY NOT NULL,
+	"document_id" integer NOT NULL,
+	"text_content" text,
+	"order_in_document" integer NOT NULL,
+	"embeddings" vector(1536),
+	"metadata" jsonb,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "content_to_space" (
+	"content_id" integer NOT NULL,
+	"space_id" integer NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "document_type" (
+	"type" text PRIMARY KEY NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "documents" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"uuid" varchar(36) NOT NULL,
+	"url" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone,
+	"type" text NOT NULL,
+	"title" text,
+	"description" text,
+	"og_image" text,
+	"raw" text,
+	"user_id" integer NOT NULL,
+	"content" text,
+	CONSTRAINT "documents_uuid_unique" UNIQUE("uuid")
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "job" (
+	"id" serial PRIMARY KEY NOT NULL,
+	"user_id" integer NOT NULL,
+	"url" text NOT NULL,
+	"status" text NOT NULL,
+	"attempts" integer DEFAULT 0 NOT NULL,
+	"lastAttemptAt" timestamp with time zone,
+	"error" text,
+	"created_at" timestamp with time zone,
+	"updated_at" timestamp with time zone
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "space_access" (
+	"space_id" integer,
+	"user_email" varchar(512),
+	"status" text
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "space_access_status" (
+	"status" text PRIMARY KEY NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "space_members" (
+	"spaceId" integer NOT NULL,
+	"user_id" integer NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "spaces" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"uuid" varchar(36) NOT NULL,
+	"name" text NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL,
+	"ownerId" integer NOT NULL,
+	"is_public" boolean DEFAULT false NOT NULL,
+	CONSTRAINT "spaces_uuid_unique" UNIQUE("uuid")
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "users" (
+	"id" serial PRIMARY KEY NOT NULL,
+	"uuid" varchar(36) NOT NULL,
+	"email" text NOT NULL,
+	"first_name" text,
+	"last_name" text,
+	"email_verified" boolean DEFAULT false NOT NULL,
+	"profile_picture_url" text,
+	"telegram_id" varchar(255),
+	"has_onboarded" integer DEFAULT 0 NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL,
+	CONSTRAINT "users_uuid_unique" UNIQUE("uuid"),
+	CONSTRAINT "users_email_unique" UNIQUE("email")
+);
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "chat_threads" ADD CONSTRAINT "chat_threads_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "chunks" ADD CONSTRAINT "chunks_document_id_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."documents"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "content_to_space" ADD CONSTRAINT "content_to_space_content_id_documents_id_fk" FOREIGN KEY ("content_id") REFERENCES "public"."documents"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "content_to_space" ADD CONSTRAINT "content_to_space_space_id_spaces_id_fk" FOREIGN KEY ("space_id") REFERENCES "public"."spaces"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "documents" ADD CONSTRAINT "documents_type_document_type_type_fk" FOREIGN KEY ("type") REFERENCES "public"."document_type"("type") ON DELETE no action ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "documents" ADD CONSTRAINT "documents_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE restrict ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "job" ADD CONSTRAINT "job_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "space_access" ADD CONSTRAINT "space_access_space_id_spaces_id_fk" FOREIGN KEY ("space_id") REFERENCES "public"."spaces"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "space_access" ADD CONSTRAINT "space_access_status_space_access_status_status_fk" FOREIGN KEY ("status") REFERENCES "public"."space_access_status"("status") ON DELETE no action ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "space_members" ADD CONSTRAINT "space_members_spaceId_users_id_fk" FOREIGN KEY ("spaceId") REFERENCES "public"."users"("id") ON DELETE restrict ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "space_members" ADD CONSTRAINT "space_members_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE restrict ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "chat_threads_user_idx" ON "chat_threads" USING btree ("user_id");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "chunk_id_idx" ON "chunks" USING btree ("id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "chunk_document_id_idx" ON "chunks" USING btree ("document_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "embeddingIndex" ON "chunks" USING diskann ("embeddings" vector_cosine_ops);--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "content_id_space_id_unique" ON "content_to_space" USING btree ("content_id","space_id");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "document_id_idx" ON "documents" USING btree ("id");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "document_uuid_idx" ON "documents" USING btree ("uuid");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "document_type_idx" ON "documents" USING btree ("type");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "document_url_user_id_idx" ON "documents" USING btree ("url","user_id");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "user_id_url_idx" ON "job" USING btree ("user_id","url");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "space_id_user_email_idx" ON "space_access" USING btree ("space_id","user_email");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "space_members_space_user_idx" ON "space_members" USING btree ("spaceId","user_id");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "spaces_id_idx" ON "spaces" USING btree ("id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "spaces_owner_id_idx" ON "spaces" USING btree ("ownerId");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "spaces_name_idx" ON "spaces" USING btree ("name");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "users_id_idx" ON "users" USING btree ("id");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "users_uuid_idx" ON "users" USING btree ("uuid");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "users_email_idx" ON "users" USING btree ("email");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "users_name_idx" ON "users" USING btree ("first_name","last_name");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "users_created_at_idx" ON "users" USING btree ("created_at");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "users_telegram_id_idx" ON "users" USING btree ("telegram_id");

apps/backend/drizzle/0001_seed-types.sql

@@ -0,0 +1,3 @@
+-- Custom SQL migration file, put you code below! --
+INSERT INTO "document_type" (type) VALUES ('tweet'), ('page'), ('note') ON CONFLICT DO NOTHING;
+INSERT INTO "document_type" (type) VALUES ('document') ON CONFLICT DO NOTHING;

apps/backend/drizzle/0002_skinny_princess_powerful.sql

@@ -0,0 +1,2 @@
+-- Active: 1732249624784@@_@5432@supermemorymain
+ALTER TABLE "documents" ADD COLUMN "is_successfully_processed" boolean DEFAULT false;

apps/backend/drizzle/0003_luxuriant_annihilus.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "documents" ALTER COLUMN "is_successfully_processed" DROP NOT NULL;--> statement-breakpoint
+ALTER TABLE "space_access" ADD COLUMN "access_type" text DEFAULT 'read' NOT NULL;--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "document_raw_user_idx" ON "documents" USING btree ("raw","user_id");--> statement-breakpoint

apps/backend/drizzle/0004_early_rick_jones.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS "saved_spaces" (
+	"user_id" integer NOT NULL,
+	"space_id" integer NOT NULL,
+	"saved_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "saved_spaces" ADD CONSTRAINT "saved_spaces_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "saved_spaces" ADD CONSTRAINT "saved_spaces_space_id_spaces_id_fk" FOREIGN KEY ("space_id") REFERENCES "public"."spaces"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "saved_spaces_user_space_idx" ON "saved_spaces" USING btree ("user_id","space_id");

apps/backend/drizzle/0005_create-access-types.sql

@@ -0,0 +1,2 @@
+-- Custom SQL migration file, put you code below! --
+INSERT INTO "space_access_status" (status) VALUES ('pending'), ('accepted'), ('rejected') ON CONFLICT DO NOTHING;

apps/backend/drizzle/0006_wandering_grandmaster.sql

@@ -0,0 +1,3 @@
+CREATE TABLE IF NOT EXISTS "waitlist" (
+	"email" varchar(512) PRIMARY KEY NOT NULL
+);

apps/backend/drizzle/0007_fantastic_serpent_society.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS "document_url_user_id_idx";

apps/backend/drizzle/0008_add-notion.sql

@@ -0,0 +1,2 @@
+-- Custom SQL migration file, put you code below! --
+INSERT INTO "document_type" (type) VALUES ('notion') ON CONFLICT DO NOTHING;

apps/backend/drizzle/0009_milky_sleepwalker.sql

@@ -0,0 +1 @@
+ALTER TABLE "waitlist" ADD COLUMN "created_at" timestamp with time zone DEFAULT now() NOT NULL;

apps/backend/drizzle/0010_heavy_preak.sql

@@ -0,0 +1,7 @@
+ALTER TABLE "documents" DROP CONSTRAINT "documents_user_id_users_id_fk";
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "documents" ADD CONSTRAINT "documents_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;

apps/backend/drizzle/0011_new_liz_osborn.sql

@@ -0,0 +1 @@
+ALTER TABLE "documents" ADD COLUMN "error_message" text;

apps/backend/drizzle/0012_small_mystique.sql

@@ -0,0 +1,2 @@
+-- Active: 1732308352274@@127.0.0.1@5432@supermemorydhravya
+ALTER TABLE "users" ADD COLUMN "last_api_key_generated_at" timestamp DEFAULT now();

apps/backend/drizzle/0013_sharp_hemingway.sql

@@ -0,0 +1 @@
+ALTER TABLE "documents" ADD COLUMN "content_hash" text;

apps/backend/drizzle/0014_mighty_the_captain.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "users" ADD COLUMN "stripe_customer_id" text;--> statement-breakpoint
+ALTER TABLE "users" ADD COLUMN "tier" text DEFAULT 'free' NOT NULL;

apps/backend/drizzle/meta/_journal.json

@@ -0,0 +1,111 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1731880515136,
+      "tag": "0000_odd_impossible_man",
+      "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "7",
+      "when": 1731880538346,
+      "tag": "0001_seed-types",
+      "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "7",
+      "when": 1731970096987,
+      "tag": "0002_skinny_princess_powerful",
+      "breakpoints": true
+    },
+    {
+      "idx": 3,
+      "version": "7",
+      "when": 1732290500234,
+      "tag": "0003_luxuriant_annihilus",
+      "breakpoints": true
+    },
+    {
+      "idx": 4,
+      "version": "7",
+      "when": 1732291518161,
+      "tag": "0004_early_rick_jones",
+      "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1732292492979,
+      "tag": "0005_create-access-types",
+      "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1732308308233,
+      "tag": "0006_wandering_grandmaster",
+      "breakpoints": true
+    },
+    {
+      "idx": 7,
+      "version": "7",
+      "when": 1732476625236,
+      "tag": "0007_fantastic_serpent_society",
+      "breakpoints": true
+    },
+    {
+      "idx": 8,
+      "version": "7",
+      "when": 1732575691923,
+      "tag": "0008_add-notion",
+      "breakpoints": true
+    },
+    {
+      "idx": 9,
+      "version": "7",
+      "when": 1732693411536,
+      "tag": "0009_milky_sleepwalker",
+      "breakpoints": true
+    },
+    {
+      "idx": 10,
+      "version": "7",
+      "when": 1733037679877,
+      "tag": "0010_heavy_preak",
+      "breakpoints": true
+    },
+    {
+      "idx": 11,
+      "version": "7",
+      "when": 1733177542412,
+      "tag": "0011_new_liz_osborn",
+      "breakpoints": true
+    },
+    {
+      "idx": 12,
+      "version": "7",
+      "when": 1735033070115,
+      "tag": "0012_small_mystique",
+      "breakpoints": true
+    },
+    {
+      "idx": 13,
+      "version": "7",
+      "when": 1736500817155,
+      "tag": "0013_sharp_hemingway",
+      "breakpoints": true
+    },
+    {
+      "idx": 14,
+      "version": "7",
+      "when": 1736852938881,
+      "tag": "0014_mighty_the_captain",
+      "breakpoints": true
+    }
+  ]
+}

apps/backend/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "supermemory-backend",
+  "scripts": {
+    "dev": "bunx wrangler -v && wrangler dev",
+    "deploy": "bunx wrangler deploy --minify",
+    "generate-migration": "dotenv -- npx drizzle-kit generate",
+    "migrate:local": "bun run ./scripts/migrate.ts",
+    "tail": "bunx wrangler tail"
+  },
+  "dependencies": {
+    "@ai-sdk/google": "^0.0.51",
+    "@ai-sdk/openai": "^0.0.70",
+    "@hono/zod-validator": "^0.4.1",
+    "@supermemory/db": "workspace:*",
+    "ai": "4.0.16",
+    "compromise": "^14.14.2",
+    "dotenv": "^16.4.5",
+    "drizzle-kit": "^0.25.0",
+    "drizzle-orm": "^0.34.1",
+    "hono": "^4.6.4",
+    "openai": "^4.68.4",
+    "postgres": "^3.4.4",
+    "uuid": "^11.0.1",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20240925.0",
+    "wrangler": "3.99.0"
+  },
+  "overrides": {
+    "iron-webcrypto": "^1.2.1"
+  }
+}

apps/backend/public/output.css

@@ -0,0 +1,853 @@
+*, ::before, ::after {
+  --tw-border-spacing-x: 0;
+  --tw-border-spacing-y: 0;
+  --tw-translate-x: 0;
+  --tw-translate-y: 0;
+  --tw-rotate: 0;
+  --tw-skew-x: 0;
+  --tw-skew-y: 0;
+  --tw-scale-x: 1;
+  --tw-scale-y: 1;
+  --tw-pan-x:  ;
+  --tw-pan-y:  ;
+  --tw-pinch-zoom:  ;
+  --tw-scroll-snap-strictness: proximity;
+  --tw-gradient-from-position:  ;
+  --tw-gradient-via-position:  ;
+  --tw-gradient-to-position:  ;
+  --tw-ordinal:  ;
+  --tw-slashed-zero:  ;
+  --tw-numeric-figure:  ;
+  --tw-numeric-spacing:  ;
+  --tw-numeric-fraction:  ;
+  --tw-ring-inset:  ;
+  --tw-ring-offset-width: 0px;
+  --tw-ring-offset-color: #fff;
+  --tw-ring-color: rgb(59 130 246 / 0.5);
+  --tw-ring-offset-shadow: 0 0 #0000;
+  --tw-ring-shadow: 0 0 #0000;
+  --tw-shadow: 0 0 #0000;
+  --tw-shadow-colored: 0 0 #0000;
+  --tw-blur:  ;
+  --tw-brightness:  ;
+  --tw-contrast:  ;
+  --tw-grayscale:  ;
+  --tw-hue-rotate:  ;
+  --tw-invert:  ;
+  --tw-saturate:  ;
+  --tw-sepia:  ;
+  --tw-drop-shadow:  ;
+  --tw-backdrop-blur:  ;
+  --tw-backdrop-brightness:  ;
+  --tw-backdrop-contrast:  ;
+  --tw-backdrop-grayscale:  ;
+  --tw-backdrop-hue-rotate:  ;
+  --tw-backdrop-invert:  ;
+  --tw-backdrop-opacity:  ;
+  --tw-backdrop-saturate:  ;
+  --tw-backdrop-sepia:  ;
+  --tw-contain-size:  ;
+  --tw-contain-layout:  ;
+  --tw-contain-paint:  ;
+  --tw-contain-style:  ;
+}
+
+::backdrop {
+  --tw-border-spacing-x: 0;
+  --tw-border-spacing-y: 0;
+  --tw-translate-x: 0;
+  --tw-translate-y: 0;
+  --tw-rotate: 0;
+  --tw-skew-x: 0;
+  --tw-skew-y: 0;
+  --tw-scale-x: 1;
+  --tw-scale-y: 1;
+  --tw-pan-x:  ;
+  --tw-pan-y:  ;
+  --tw-pinch-zoom:  ;
+  --tw-scroll-snap-strictness: proximity;
+  --tw-gradient-from-position:  ;
+  --tw-gradient-via-position:  ;
+  --tw-gradient-to-position:  ;
+  --tw-ordinal:  ;
+  --tw-slashed-zero:  ;
+  --tw-numeric-figure:  ;
+  --tw-numeric-spacing:  ;
+  --tw-numeric-fraction:  ;
+  --tw-ring-inset:  ;
+  --tw-ring-offset-width: 0px;
+  --tw-ring-offset-color: #fff;
+  --tw-ring-color: rgb(59 130 246 / 0.5);
+  --tw-ring-offset-shadow: 0 0 #0000;
+  --tw-ring-shadow: 0 0 #0000;
+  --tw-shadow: 0 0 #0000;
+  --tw-shadow-colored: 0 0 #0000;
+  --tw-blur:  ;
+  --tw-brightness:  ;
+  --tw-contrast:  ;
+  --tw-grayscale:  ;
+  --tw-hue-rotate:  ;
+  --tw-invert:  ;
+  --tw-saturate:  ;
+  --tw-sepia:  ;
+  --tw-drop-shadow:  ;
+  --tw-backdrop-blur:  ;
+  --tw-backdrop-brightness:  ;
+  --tw-backdrop-contrast:  ;
+  --tw-backdrop-grayscale:  ;
+  --tw-backdrop-hue-rotate:  ;
+  --tw-backdrop-invert:  ;
+  --tw-backdrop-opacity:  ;
+  --tw-backdrop-saturate:  ;
+  --tw-backdrop-sepia:  ;
+  --tw-contain-size:  ;
+  --tw-contain-layout:  ;
+  --tw-contain-paint:  ;
+  --tw-contain-style:  ;
+}
+
+/*
+! tailwindcss v3.4.15 | MIT License | https://tailwindcss.com
+*/
+
+/*
+1. Prevent padding and border from affecting element width. (https://github.com/mozdevs/cssremedy/issues/4)
+2. Allow adding a border to an element by just adding a border-width. (https://github.com/tailwindcss/tailwindcss/pull/116)
+*/
+
+*,
+::before,
+::after {
+  box-sizing: border-box;
+  /* 1 */
+  border-width: 0;
+  /* 2 */
+  border-style: solid;
+  /* 2 */
+  border-color: #e5e7eb;
+  /* 2 */
+}
+
+::before,
+::after {
+  --tw-content: '';
+}
+
+/*
+1. Use a consistent sensible line-height in all browsers.
+2. Prevent adjustments of font size after orientation changes in iOS.
+3. Use a more readable tab size.
+4. Use the user's configured `sans` font-family by default.
+5. Use the user's configured `sans` font-feature-settings by default.
+6. Use the user's configured `sans` font-variation-settings by default.
+7. Disable tap highlights on iOS
+*/
+
+html,
+:host {
+  line-height: 1.5;
+  /* 1 */
+  -webkit-text-size-adjust: 100%;
+  /* 2 */
+  -moz-tab-size: 4;
+  /* 3 */
+  -o-tab-size: 4;
+     tab-size: 4;
+  /* 3 */
+  font-family: ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+  /* 4 */
+  font-feature-settings: normal;
+  /* 5 */
+  font-variation-settings: normal;
+  /* 6 */
+  -webkit-tap-highlight-color: transparent;
+  /* 7 */
+}
+
+/*
+1. Remove the margin in all browsers.
+2. Inherit line-height from `html` so users can set them as a class directly on the `html` element.
+*/
+
+body {
+  margin: 0;
+  /* 1 */
+  line-height: inherit;
+  /* 2 */
+}
+
+/*
+1. Add the correct height in Firefox.
+2. Correct the inheritance of border color in Firefox. (https://bugzilla.mozilla.org/show_bug.cgi?id=190655)
+3. Ensure horizontal rules are visible by default.
+*/
+
+hr {
+  height: 0;
+  /* 1 */
+  color: inherit;
+  /* 2 */
+  border-top-width: 1px;
+  /* 3 */
+}
+
+/*
+Add the correct text decoration in Chrome, Edge, and Safari.
+*/
+
+abbr:where([title]) {
+  -webkit-text-decoration: underline dotted;
+          text-decoration: underline dotted;
+}
+
+/*
+Remove the default font size and weight for headings.
+*/
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  font-size: inherit;
+  font-weight: inherit;
+}
+
+/*
+Reset links to optimize for opt-in styling instead of opt-out.
+*/
+
+a {
+  color: inherit;
+  text-decoration: inherit;
+}
+
+/*
+Add the correct font weight in Edge and Safari.
+*/
+
+b,
+strong {
+  font-weight: bolder;
+}
+
+/*
+1. Use the user's configured `mono` font-family by default.
+2. Use the user's configured `mono` font-feature-settings by default.
+3. Use the user's configured `mono` font-variation-settings by default.
+4. Correct the odd `em` font sizing in all browsers.
+*/
+
+code,
+kbd,
+samp,
+pre {
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+  /* 1 */
+  font-feature-settings: normal;
+  /* 2 */
+  font-variation-settings: normal;
+  /* 3 */
+  font-size: 1em;
+  /* 4 */
+}
+
+/*
+Add the correct font size in all browsers.
+*/
+
+small {
+  font-size: 80%;
+}
+
+/*
+Prevent `sub` and `sup` elements from affecting the line height in all browsers.
+*/
+
+sub,
+sup {
+  font-size: 75%;
+  line-height: 0;
+  position: relative;
+  vertical-align: baseline;
+}
+
+sub {
+  bottom: -0.25em;
+}
+
+sup {
+  top: -0.5em;
+}
+
+/*
+1. Remove text indentation from table contents in Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=999088, https://bugs.webkit.org/show_bug.cgi?id=201297)
+2. Correct table border color inheritance in all Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=935729, https://bugs.webkit.org/show_bug.cgi?id=195016)
+3. Remove gaps between table borders by default.
+*/
+
+table {
+  text-indent: 0;
+  /* 1 */
+  border-color: inherit;
+  /* 2 */
+  border-collapse: collapse;
+  /* 3 */
+}
+
+/*
+1. Change the font styles in all browsers.
+2. Remove the margin in Firefox and Safari.
+3. Remove default padding in all browsers.
+*/
+
+button,
+input,
+optgroup,
+select,
+textarea {
+  font-family: inherit;
+  /* 1 */
+  font-feature-settings: inherit;
+  /* 1 */
+  font-variation-settings: inherit;
+  /* 1 */
+  font-size: 100%;
+  /* 1 */
+  font-weight: inherit;
+  /* 1 */
+  line-height: inherit;
+  /* 1 */
+  letter-spacing: inherit;
+  /* 1 */
+  color: inherit;
+  /* 1 */
+  margin: 0;
+  /* 2 */
+  padding: 0;
+  /* 3 */
+}
+
+/*
+Remove the inheritance of text transform in Edge and Firefox.
+*/
+
+button,
+select {
+  text-transform: none;
+}
+
+/*
+1. Correct the inability to style clickable types in iOS and Safari.
+2. Remove default button styles.
+*/
+
+button,
+input:where([type='button']),
+input:where([type='reset']),
+input:where([type='submit']) {
+  -webkit-appearance: button;
+  /* 1 */
+  background-color: transparent;
+  /* 2 */
+  background-image: none;
+  /* 2 */
+}
+
+/*
+Use the modern Firefox focus style for all focusable elements.
+*/
+
+:-moz-focusring {
+  outline: auto;
+}
+
+/*
+Remove the additional `:invalid` styles in Firefox. (https://github.com/mozilla/gecko-dev/blob/2f9eacd9d3d995c937b4251a5557d95d494c9be1/layout/style/res/forms.css#L728-L737)
+*/
+
+:-moz-ui-invalid {
+  box-shadow: none;
+}
+
+/*
+Add the correct vertical alignment in Chrome and Firefox.
+*/
+
+progress {
+  vertical-align: baseline;
+}
+
+/*
+Correct the cursor style of increment and decrement buttons in Safari.
+*/
+
+::-webkit-inner-spin-button,
+::-webkit-outer-spin-button {
+  height: auto;
+}
+
+/*
+1. Correct the odd appearance in Chrome and Safari.
+2. Correct the outline style in Safari.
+*/
+
+[type='search'] {
+  -webkit-appearance: textfield;
+  /* 1 */
+  outline-offset: -2px;
+  /* 2 */
+}
+
+/*
+Remove the inner padding in Chrome and Safari on macOS.
+*/
+
+::-webkit-search-decoration {
+  -webkit-appearance: none;
+}
+
+/*
+1. Correct the inability to style clickable types in iOS and Safari.
+2. Change font properties to `inherit` in Safari.
+*/
+
+::-webkit-file-upload-button {
+  -webkit-appearance: button;
+  /* 1 */
+  font: inherit;
+  /* 2 */
+}
+
+/*
+Add the correct display in Chrome and Safari.
+*/
+
+summary {
+  display: list-item;
+}
+
+/*
+Removes the default spacing and border for appropriate elements.
+*/
+
+blockquote,
+dl,
+dd,
+h1,
+h2,
+h3,
+h4,
+h5,
+h6,
+hr,
+figure,
+p,
+pre {
+  margin: 0;
+}
+
+fieldset {
+  margin: 0;
+  padding: 0;
+}
+
+legend {
+  padding: 0;
+}
+
+ol,
+ul,
+menu {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+}
+
+/*
+Reset default styling for dialogs.
+*/
+
+dialog {
+  padding: 0;
+}
+
+/*
+Prevent resizing textareas horizontally by default.
+*/
+
+textarea {
+  resize: vertical;
+}
+
+/*
+1. Reset the default placeholder opacity in Firefox. (https://github.com/tailwindlabs/tailwindcss/issues/3300)
+2. Set the default placeholder color to the user's configured gray 400 color.
+*/
+
+input::-moz-placeholder, textarea::-moz-placeholder {
+  opacity: 1;
+  /* 1 */
+  color: #9ca3af;
+  /* 2 */
+}
+
+input::placeholder,
+textarea::placeholder {
+  opacity: 1;
+  /* 1 */
+  color: #9ca3af;
+  /* 2 */
+}
+
+/*
+Set the default cursor for buttons.
+*/
+
+button,
+[role="button"] {
+  cursor: pointer;
+}
+
+/*
+Make sure disabled buttons don't get the pointer cursor.
+*/
+
+:disabled {
+  cursor: default;
+}
+
+/*
+1. Make replaced elements `display: block` by default. (https://github.com/mozdevs/cssremedy/issues/14)
+2. Add `vertical-align: middle` to align replaced elements more sensibly by default. (https://github.com/jensimmons/cssremedy/issues/14#issuecomment-634934210)
+   This can trigger a poorly considered lint error in some tools but is included by design.
+*/
+
+img,
+svg,
+video,
+canvas,
+audio,
+iframe,
+embed,
+object {
+  display: block;
+  /* 1 */
+  vertical-align: middle;
+  /* 2 */
+}
+
+/*
+Constrain images and videos to the parent width and preserve their intrinsic aspect ratio. (https://github.com/mozdevs/cssremedy/issues/14)
+*/
+
+img,
+video {
+  max-width: 100%;
+  height: auto;
+}
+
+/* Make elements with the HTML hidden attribute stay hidden by default */
+
+[hidden]:where(:not([hidden="until-found"])) {
+  display: none;
+}
+
+.container {
+  width: 100%;
+}
+
+@media (min-width: 640px) {
+  .container {
+    max-width: 640px;
+  }
+}
+
+@media (min-width: 768px) {
+  .container {
+    max-width: 768px;
+  }
+}
+
+@media (min-width: 1024px) {
+  .container {
+    max-width: 1024px;
+  }
+}
+
+@media (min-width: 1280px) {
+  .container {
+    max-width: 1280px;
+  }
+}
+
+@media (min-width: 1536px) {
+  .container {
+    max-width: 1536px;
+  }
+}
+
+.mx-auto {
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.mb-12 {
+  margin-bottom: 3rem;
+}
+
+.mb-4 {
+  margin-bottom: 1rem;
+}
+
+.mb-8 {
+  margin-bottom: 2rem;
+}
+
+.mt-8 {
+  margin-top: 2rem;
+}
+
+.block {
+  display: block;
+}
+
+.flex {
+  display: flex;
+}
+
+.table {
+  display: table;
+}
+
+.grid {
+  display: grid;
+}
+
+.hidden {
+  display: none;
+}
+
+.transform {
+  transform: translate(var(--tw-translate-x), var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y));
+}
+
+.items-center {
+  align-items: center;
+}
+
+.justify-center {
+  justify-content: center;
+}
+
+.justify-between {
+  justify-content: space-between;
+}
+
+.gap-8 {
+  gap: 2rem;
+}
+
+.space-x-4 > :not([hidden]) ~ :not([hidden]) {
+  --tw-space-x-reverse: 0;
+  margin-right: calc(1rem * var(--tw-space-x-reverse));
+  margin-left: calc(1rem * calc(1 - var(--tw-space-x-reverse)));
+}
+
+.space-x-8 > :not([hidden]) ~ :not([hidden]) {
+  --tw-space-x-reverse: 0;
+  margin-right: calc(2rem * var(--tw-space-x-reverse));
+  margin-left: calc(2rem * calc(1 - var(--tw-space-x-reverse)));
+}
+
+.space-y-2 > :not([hidden]) ~ :not([hidden]) {
+  --tw-space-y-reverse: 0;
+  margin-top: calc(0.5rem * calc(1 - var(--tw-space-y-reverse)));
+  margin-bottom: calc(0.5rem * var(--tw-space-y-reverse));
+}
+
+.rounded-lg {
+  border-radius: 0.5rem;
+}
+
+.border {
+  border-width: 1px;
+}
+
+.border-t {
+  border-top-width: 1px;
+}
+
+.border-gray-600 {
+  --tw-border-opacity: 1;
+  border-color: rgb(75 85 99 / var(--tw-border-opacity, 1));
+}
+
+.border-gray-800 {
+  --tw-border-opacity: 1;
+  border-color: rgb(31 41 55 / var(--tw-border-opacity, 1));
+}
+
+.bg-black\/50 {
+  background-color: rgb(0 0 0 / 0.5);
+}
+
+.bg-blue-600 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(37 99 235 / var(--tw-bg-opacity, 1));
+}
+
+.bg-gray-900 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(17 24 39 / var(--tw-bg-opacity, 1));
+}
+
+.bg-gray-900\/50 {
+  background-color: rgb(17 24 39 / 0.5);
+}
+
+.p-6 {
+  padding: 1.5rem;
+}
+
+.px-6 {
+  padding-left: 1.5rem;
+  padding-right: 1.5rem;
+}
+
+.px-8 {
+  padding-left: 2rem;
+  padding-right: 2rem;
+}
+
+.py-16 {
+  padding-top: 4rem;
+  padding-bottom: 4rem;
+}
+
+.py-2 {
+  padding-top: 0.5rem;
+  padding-bottom: 0.5rem;
+}
+
+.py-20 {
+  padding-top: 5rem;
+  padding-bottom: 5rem;
+}
+
+.py-3 {
+  padding-top: 0.75rem;
+  padding-bottom: 0.75rem;
+}
+
+.py-4 {
+  padding-top: 1rem;
+  padding-bottom: 1rem;
+}
+
+.pt-8 {
+  padding-top: 2rem;
+}
+
+.text-center {
+  text-align: center;
+}
+
+.text-2xl {
+  font-size: 1.5rem;
+  line-height: 2rem;
+}
+
+.text-3xl {
+  font-size: 1.875rem;
+  line-height: 2.25rem;
+}
+
+.text-4xl {
+  font-size: 2.25rem;
+  line-height: 2.5rem;
+}
+
+.text-lg {
+  font-size: 1.125rem;
+  line-height: 1.75rem;
+}
+
+.text-xl {
+  font-size: 1.25rem;
+  line-height: 1.75rem;
+}
+
+.font-bold {
+  font-weight: 700;
+}
+
+.font-semibold {
+  font-weight: 600;
+}
+
+.text-gray-300 {
+  --tw-text-opacity: 1;
+  color: rgb(209 213 219 / var(--tw-text-opacity, 1));
+}
+
+.text-gray-400 {
+  --tw-text-opacity: 1;
+  color: rgb(156 163 175 / var(--tw-text-opacity, 1));
+}
+
+.text-white {
+  --tw-text-opacity: 1;
+  color: rgb(255 255 255 / var(--tw-text-opacity, 1));
+}
+
+.filter {
+  filter: var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow);
+}
+
+.transition {
+  transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, -webkit-backdrop-filter;
+  transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter;
+  transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter, -webkit-backdrop-filter;
+  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
+  transition-duration: 150ms;
+}
+
+.duration-300 {
+  transition-duration: 300ms;
+}
+
+.hover\:bg-blue-700:hover {
+  --tw-bg-opacity: 1;
+  background-color: rgb(29 78 216 / var(--tw-bg-opacity, 1));
+}
+
+.hover\:bg-gray-700:hover {
+  --tw-bg-opacity: 1;
+  background-color: rgb(55 65 81 / var(--tw-bg-opacity, 1));
+}
+
+.hover\:bg-gray-800:hover {
+  --tw-bg-opacity: 1;
+  background-color: rgb(31 41 55 / var(--tw-bg-opacity, 1));
+}
+
+.hover\:text-white:hover {
+  --tw-text-opacity: 1;
+  color: rgb(255 255 255 / var(--tw-text-opacity, 1));
+}
+
+@media (min-width: 768px) {
+  .md\:flex {
+    display: flex;
+  }
+
+  .md\:grid-cols-3 {
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+  }
+
+  .md\:grid-cols-4 {
+    grid-template-columns: repeat(4, minmax(0, 1fr));
+  }
+}

apps/backend/scripts/migrate.ts

@@ -0,0 +1,36 @@
+import { config } from "dotenv";
+import { drizzle } from "drizzle-orm/postgres-js";
+import { migrate } from "drizzle-orm/postgres-js/migrator";
+import process from "node:process";
+import postgres from "postgres";
+
+config();
+
+if (!process.env.DATABASE_URL) {
+	throw new Error("DATABASE_URL is not set");
+}
+
+const connectionString = process.env.DATABASE_URL!;
+
+console.log("Connecting to:", connectionString.replace(/:[^:@]+@/, ":****@")); // Log sanitized connection string
+
+const migrationClient = postgres(connectionString, { max: 1 });
+
+async function main() {
+	console.log("Running migrations...");
+
+	try {
+		const db = drizzle(migrationClient);
+		await migrate(db, { migrationsFolder: "./drizzle" });
+		console.log("Migrations completed!");
+	} catch (error) {
+		console.error("Migration failed:", error);
+	} finally {
+		await migrationClient.end();
+	}
+}
+
+main().catch((err) => {
+	console.error("Unexpected error:", err);
+	process.exit(1);
+});

apps/backend/src/auth.ts

@@ -0,0 +1,151 @@
+import { Context, Next } from "hono";
+import { getSessionFromRequest } from "@supermemory/authkit-remix-cloudflare/src/session";
+import { and, database, eq, sql } from "@supermemory/db";
+import { User, users } from "@supermemory/db/schema";
+import { Env, Variables } from "./types";
+import { encrypt, decrypt } from "./utils/cipher";
+
+interface EncryptedData {
+  userId: string;
+  lastApiKeyGeneratedAt: string;
+}
+
+export const getApiKey = async (
+  userId: string,
+  lastApiKeyGeneratedAt: string,
+  c: Context<{ Variables: Variables; Bindings: Env }>
+) => {
+  const data = `${userId}-${lastApiKeyGeneratedAt}`;
+  return "sm_" + (await encrypt(data, c.env.WORKOS_COOKIE_PASSWORD));
+};
+
+export const decryptApiKey = async (
+  encryptedKey: string,
+  c: Context<{ Variables: Variables; Bindings: Env }>
+): Promise<EncryptedData> => {
+  const ourKey = encryptedKey.slice(3);
+  const decrypted = await decrypt(ourKey, c.env.WORKOS_COOKIE_PASSWORD);
+  const [userId, lastApiKeyGeneratedAt] = decrypted.split("-");
+
+  return {
+    userId,
+    lastApiKeyGeneratedAt,
+  };
+};
+
+export const auth = async (
+  c: Context<{ Variables: Variables; Bindings: Env }>,
+  next: Next
+) => {
+  // Handle CORS preflight requests
+  if (c.req.method === "OPTIONS") {
+    return next()
+  }
+
+  // Set cache control headers
+  c.header("Cache-Control", "private, no-cache, no-store, must-revalidate");
+  c.header("Pragma", "no-cache");
+  c.header("Expires", "0");
+
+  let user: User | User[] | undefined;
+
+  // Check for API key authentication first
+  const authHeader = c.req.raw.headers.get("Authorization");
+  if (authHeader?.startsWith("Bearer ")) {
+    const apiKey = authHeader.slice(7);
+    try {
+      const { userId, lastApiKeyGeneratedAt } = await decryptApiKey(apiKey, c);
+      
+      // Look up user with matching id and lastApiKeyGeneratedAt
+      user = await database(c.env.HYPERDRIVE.connectionString)
+        .select()
+        .from(users)
+        .where(
+          and(
+            eq(users.uuid, userId)
+          )
+        )
+        .limit(1);
+
+      if (user && Array.isArray(user)) {
+        user = user[0];
+        if (user && user.lastApiKeyGeneratedAt?.getTime() === Number(lastApiKeyGeneratedAt)) {
+          c.set("user", user);
+        } else {
+          return c.json({ error: "Invalid API key - user not found" }, 401);
+        }
+      }
+    } catch (err) {
+      console.error("API key authentication failed:", err);
+      return c.json({ error: "Invalid API key format" }, 401);
+    }
+  }
+
+  // If no user found via API key, try cookie authentication
+  if (!user) {
+    const cookies = c.req.raw.headers.get("Cookie");
+    if (cookies) {
+      // Fake remix context object. this just works.
+      const context = {
+        cloudflare: {
+          env: c.env,
+        },
+      };
+
+      const session = await getSessionFromRequest(c.req.raw, context);
+      console.log("Session", session);
+      c.set("session", session);
+
+      if (session?.user?.id) {
+        user = await database(c.env.HYPERDRIVE.connectionString)
+          .select()
+          .from(users)
+          .where(eq(users.uuid, session.user.id))
+          .limit(1);
+
+        if ((!user || user.length === 0) && session?.user?.id) {
+          const newUser = await database(c.env.HYPERDRIVE.connectionString)
+            .insert(users)
+            .values({
+              uuid: session.user?.id,
+              email: session.user?.email,
+              firstName: session.user?.firstName,
+              lastName: session.user?.lastName,
+              createdAt: new Date(),
+              updatedAt: new Date(),
+              emailVerified: false,
+              profilePictureUrl: session.user?.profilePictureUrl ?? "",
+            })
+            .returning()
+            .onConflictDoUpdate({
+              target: [users.email],
+              set: {
+                uuid: session.user.id,
+              },
+            });
+
+          user = newUser[0];
+        }
+
+        user = Array.isArray(user) ? user[0] : user;
+        c.set("user", user);
+        console.log("User", user);
+      }
+    }
+  }
+
+  // Check if request requires authentication
+  const isPublicSpaceRequest =
+    c.req.url.includes("/api/spaces/") || c.req.url.includes("/api/memories");
+
+  if (!isPublicSpaceRequest && !c.get("user")) {
+    console.log("Unauthorized access to", c.req.url);
+    if (authHeader) {
+      return c.json({ error: "Invalid authentication credentials" }, 401);
+    } else {
+      return c.json({ error: "Authentication required" }, 401);
+    }
+  }
+
+  return next();
+};

apps/backend/src/components/landing.tsx

@@ -0,0 +1,234 @@
+import { html } from "hono/html";
+
+export function LandingPage() {
+  return (
+    <html lang="en">
+      <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <link href="/output.css" rel="stylesheet" />
+        <title>Supermemory API</title>
+      </head>
+      <body>
+        <div className="gradient-dark">
+          <header className="bg-gray-900/50 dot-pattern">
+            <nav className="container mx-auto px-6 py-4">
+              <div className="flex items-center justify-between">
+                <div className="text-2xl font-bold text-white">
+                  Supermemory API
+                </div>
+                <div className="hidden md:flex space-x-8">
+                  <a
+                    href="#features"
+                    className="text-gray-300 hover:text-white"
+                  >
+                    Features
+                  </a>
+                  <a
+                    href="https://docs.supermemory.ai/"
+                    target="_blank"
+                    className="text-gray-300 hover:text-white"
+                    rel="noreferrer"
+                  >
+                    Documentation
+                  </a>
+                </div>
+                <a
+                  href="https://docs.supermemory.ai/"
+                  target="_blank"
+                  className="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700"
+                  rel="noreferrer"
+                >
+                  Get Started
+                </a>
+              </div>
+            </nav>
+
+            <div className="container mx-auto px-6 py-16 text-center">
+              <h1 className="text-4xl font-bold text-white mb-4">
+                The Modern API for Knowledge Management
+              </h1>
+              <p className="text-xl text-gray-300 mb-8">
+                Build powerful search and AI applications with our flexible,
+                production-ready API
+              </p>
+              <div className="flex justify-center space-x-4">
+                <a
+                  href="https://docs.supermemory.ai/"
+                  target="_blank"
+                  className="bg-blue-600 text-white px-8 py-3 rounded-lg hover:bg-blue-700"
+                  rel="noreferrer"
+                >
+                  Get Started Free
+                </a>
+                <a
+                  href="https://docs.supermemory.ai/"
+                  target="_blank"
+                  className="border border-gray-600 text-gray-300 px-8 py-3 rounded-lg hover:bg-gray-700"
+                  rel="noreferrer"
+                >
+                  View Docs
+                </a>
+              </div>
+            </div>
+          </header>
+
+          <section id="features" className="py-20 bg-gray-900/50 dot-pattern">
+            <div className="container mx-auto px-6">
+              <h2 className="text-3xl font-bold text-center text-white mb-12">
+                Key Features
+              </h2>
+              <div className="grid md:grid-cols-3 gap-8">
+                <div className="p-6 border border-gray-600 rounded-lg bg-gray-900 hover:bg-gray-800 transition duration-300">
+                  <h3 className="text-xl font-semibold mb-4 text-white">
+                    Battle-Tested RAG Stack
+                  </h3>
+                  <p className="text-gray-300">
+                    Production-ready retrieval augmented generation architecture
+                    for reliable and scalable information retrieval.
+                  </p>
+                </div>
+                <div className="p-6 border border-gray-600 rounded-lg bg-gray-900 hover:bg-gray-800 transition duration-300">
+                  <h3 className="text-xl font-semibold mb-4 text-white">
+                    Flexible LLM Integration
+                  </h3>
+                  <p className="text-gray-300">
+                    Use any LLM of your choice or operate in search-only mode
+                    for maximum flexibility and control.
+                  </p>
+                </div>
+                <div className="p-6 border border-gray-600 rounded-lg bg-gray-900 hover:bg-gray-800 transition duration-300">
+                  <h3 className="text-xl font-semibold mb-4 text-white">
+                    Advanced Access Control
+                  </h3>
+                  <p className="text-gray-300">
+                    Comprehensive collection filtering and permission management
+                    for secure data access.
+                  </p>
+                </div>
+                <div className="p-6 border border-gray-600 rounded-lg bg-gray-900 hover:bg-gray-800 transition duration-300">
+                  <h3 className="text-xl font-semibold mb-4 text-white">
+                    Seamless Data Import
+                  </h3>
+                  <p className="text-gray-300">
+                    Magic link import and platform synchronization for
+                    effortless data integration.
+                  </p>
+                </div>
+                <div className="p-6 border border-gray-600 rounded-lg bg-gray-900 hover:bg-gray-800 transition duration-300">
+                  <h3 className="text-xl font-semibold mb-4 text-white">
+                    Real-time Monitoring
+                  </h3>
+                  <p className="text-gray-300">
+                    Track and analyze memory usage patterns in real-time with
+                    detailed metrics.
+                  </p>
+                </div>
+                <div className="p-6 border border-gray-600 rounded-lg bg-gray-900 hover:bg-gray-800 transition duration-300">
+                  <h3 className="text-xl font-semibold mb-4 text-white">
+                    Easy Integration
+                  </h3>
+                  <p className="text-gray-300">
+                    Simple API endpoints that integrate seamlessly with your
+                    existing infrastructure.
+                  </p>
+                </div>
+              </div>
+            </div>
+          </section>
+
+          <footer className="bg-black/50 dot-pattern">
+            <div className="container mx-auto px-6">
+              <div className="grid md:grid-cols-4 gap-8">
+                <div>
+                  <h4 className="text-lg font-semibold mb-4">
+                    Supermemory API
+                  </h4>
+                  <p className="text-gray-400">
+                    Making memory management simple and efficient for developers
+                    worldwide.
+                  </p>
+                </div>
+                <div>
+                  <h4 className="text-lg font-semibold mb-4">Product</h4>
+                  <ul className="space-y-2 text-gray-400">
+                    <li>
+                      <a href="#features" className="hover:text-white">
+                        Features
+                      </a>
+                    </li>
+                    <li>
+                      <a
+                        href="https://docs.supermemory.ai/"
+                        target="_blank"
+                        className="hover:text-white"
+                        rel="noreferrer"
+                      >
+                        Documentation
+                      </a>
+                    </li>
+                  </ul>
+                </div>
+                <div>
+                  <h4 className="text-lg font-semibold mb-4">Connect</h4>
+                  <ul className="space-y-2 text-gray-400">
+                    <li>
+                      <a
+                        href="https://x.com/supermemoryai"
+                        target="_blank"
+                        className="hover:text-white"
+                        rel="noreferrer"
+                      >
+                        X (formerly Twitter)
+                      </a>
+                    </li>
+                    <li>
+                      <a
+                        href="https://github.com/supermemoryai"
+                        target="_blank"
+                        className="hover:text-white"
+                        rel="noreferrer"
+                      >
+                        GitHub
+                      </a>
+                    </li>
+                    <li>
+                      <a
+                        href="https://discord.gg/supermemory"
+                        target="_blank"
+                        className="hover:text-white"
+                        rel="noreferrer"
+                      >
+                        Discord
+                      </a>
+                    </li>
+                  </ul>
+                </div>
+              </div>
+              <div className="border-t border-gray-800 mt-8 pt-8 text-center text-gray-400">
+                <p>&copy; 2024 Supermemory API. All rights reserved.</p>
+              </div>
+            </div>
+          </footer>
+
+          <style
+            dangerouslySetInnerHTML={{
+              __html: `
+				.dot-pattern {
+					background-image: radial-gradient(
+						rgba(255, 255, 255, 0.1) 1px,
+						transparent 1px
+					);
+					background-size: 24px 24px;
+				}
+				.gradient-dark {
+					background: linear-gradient(to bottom right, rgb(17 24 39), rgb(0 0 0));
+				}
+			`,
+            }}
+          />
+        </div>
+      </body>
+    </html>
+  );
+}

apps/backend/src/errors/baseError.ts

@@ -42,5 +42,4 @@ export class BaseHttpError extends Error {
         source: this.source,
       };
     }
-  }
-  
+  }

apps/backend/src/errors/results.ts

@@ -28,4 +28,4 @@ export function isErr<T, E extends Error>(
 	result: Result<T, E>,
 ): result is { ok: false; error: E } {
 	return !result.ok;
-}
+}

apps/backend/src/globals.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

apps/backend/src/index.tsx

@@ -0,0 +1,221 @@
+import { z } from "zod";
+import { Context, Hono } from "hono";
+import { auth } from "./auth";
+import { logger } from "hono/logger";
+import { timing } from "hono/timing";
+import { Env, Variables } from "./types";
+import { zValidator } from "@hono/zod-validator";
+import { database } from "@supermemory/db";
+import { waitlist } from "@supermemory/db/schema";
+import { cors } from "hono/cors";
+import { ContentWorkflow } from "./workflow";
+import { Resend } from "resend";
+import { StatusCode } from "hono/utils/http-status";
+import { LandingPage } from "./components/landing";
+import user from "./routes/user";
+import spacesRoute from "./routes/spaces";
+import actions from "./routes/actions";
+import memories from "./routes/memories";
+import integrations from "./routes/integrations";
+import {
+  cloudflareRateLimiter,
+  DurableObjectRateLimiter,
+  DurableObjectStore,
+} from "@hono-rate-limiter/cloudflare";
+import { ConfigType, GeneralConfigType, rateLimiter } from "hono-rate-limiter";
+
+export const app = new Hono<{ Variables: Variables; Bindings: Env }>()
+  .use("*", timing())
+  .use("*", logger())
+  .use(
+    "*",
+    cors({
+      origin: [
+        "http://localhost:3000",
+        "https://supermemory.ai",
+        "https://*.supermemory.ai",
+        "https://*.supermemory.com",
+        "https://supermemory.com",
+        "chrome-extension://*",
+      ],
+      allowHeaders: ["*"],
+      allowMethods: ["*"],
+      credentials: true,
+      exposeHeaders: ["*"],
+    })
+  )
+  .use("/api/*", auth)
+  .use("/api/*", (c, next) => {
+    const user = c.get("user");
+
+    // RATELIMITS
+    const rateLimitConfig = {
+      // Endpoints that bypass rate limiting
+      excludedPaths: [
+        "/api/add",
+        "/api/chat",
+        "/api/suggested-learnings",
+        "/api/recommended-questions",
+      ] as (string | RegExp)[],
+
+      // Custom rate limits for specific endpoints
+      customLimits: {
+        notionImport: {
+          paths: [
+            "/api/integrations/notion/import",
+            "/api/integrations/notion",
+          ],
+          windowMs: 10 * 60 * 1000, // 10 minutes
+          limit: 5, // 5 requests per 10 minutes
+        },
+        inviteSpace: {
+          paths: [/^\/api\/spaces\/[^/]+\/invite$/],
+          windowMs: 60 * 1000, // 1 minute
+          limit: 5, // 5 requests per minute
+        },
+      } as Record<
+        string,
+        { paths: (string | RegExp)[]; windowMs: number; limit: number }
+      >,
+
+      default: {
+        windowMs: 60 * 1000, // 1 minute
+        limit: 100, // 100 requests per minute
+      },
+
+      common: {
+        standardHeaders: "draft-6",
+        keyGenerator: (c: Context) =>
+          (user?.uuid ?? c.req.header("cf-connecting-ip")) +
+          "-" +
+          new Date().getDate(), // day so that limit gets reset every day
+        store: new DurableObjectStore({ namespace: c.env.RATE_LIMITER }),
+      } as GeneralConfigType<ConfigType>,
+    };
+
+    if (
+      c.req.path &&
+      rateLimitConfig.excludedPaths.some((path) =>
+        typeof path === "string" ? c.req.path === path : path.test(c.req.path)
+      )
+    ) {
+      return next();
+    }
+
+    // Check for custom rate limits
+    for (const [_, config] of Object.entries(rateLimitConfig.customLimits)) {
+      if (
+        config.paths.some((path) =>
+          typeof path === "string" ? c.req.path === path : path.test(c.req.path)
+        )
+      ) {
+        return rateLimiter({
+          windowMs: config.windowMs,
+          limit: config.limit,
+          ...rateLimitConfig.common,
+        })(c as any, next);
+      }
+    }
+
+    // Apply default rate limit
+    return rateLimiter({
+      windowMs: rateLimitConfig.default.windowMs,
+      limit: rateLimitConfig.default.limit,
+      ...rateLimitConfig.common,
+    })(c as any, next);
+  })
+  .get("/", (c) => {
+    return c.html(<LandingPage />);
+  })
+  .route("/api/user", user)
+  .route("/api/spaces", spacesRoute)
+  .route("/api", actions)
+  .route("/api/integrations", integrations)
+  .route("/api/memories", memories)
+  .post(
+    "/waitlist",
+    zValidator(
+      "json",
+      z.object({ email: z.string().email(), token: z.string() })
+    ),
+    async (c) => {
+      const { email, token } = c.req.valid("json");
+
+      const address = c.req.raw.headers.get("CF-Connecting-IP");
+
+      const idempotencyKey = crypto.randomUUID();
+      const url = "https://challenges.cloudflare.com/turnstile/v0/siteverify";
+      const firstResult = await fetch(url, {
+        body: JSON.stringify({
+          secret: c.env.TURNSTILE_SECRET_KEY,
+          response: token,
+          remoteip: address,
+          idempotency_key: idempotencyKey,
+        }),
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+      });
+
+      const firstOutcome = (await firstResult.json()) as { success: boolean };
+
+      if (!firstOutcome.success) {
+        console.info("Turnstile verification failed", firstOutcome);
+        return c.json(
+          { error: "Turnstile verification failed" },
+          439 as StatusCode
+        );
+      }
+
+      const resend = new Resend(c.env.RESEND_API_KEY);
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      const ip =
+        c.req.header("cf-connecting-ip") ||
+        `${c.req.raw.cf?.asn}-${c.req.raw.cf?.country}-${c.req.raw.cf?.city}-${c.req.raw.cf?.region}-${c.req.raw.cf?.postalCode}`;
+
+      const { success } = await c.env.EMAIL_LIMITER.limit({ key: ip });
+
+      if (!success) {
+        return c.json({ error: "Rate limit exceeded" }, 429);
+      }
+
+      const message = `Supermemory started as a side project a few months ago when I built it as a hackathon project.
+    <br></br>
+    you guys loved it too much. like wayy too much. it was embarrassing, because this was not it - it was nothing but a hackathon project.
+    <br></br>
+    I launched on github too. <a href="https://git.new/memory">https://github.com/supermemoryai/supermemory</a>, and we were somehow one of the fastest growing open source repositories in Q3 2024.
+   <br></br><br></br>
+    So, it's time to make this good. My vision is to make supermemory the best memory tool on the internet.
+    `;
+
+      try {
+        await db.insert(waitlist).values({ email });
+        await resend.emails.send({
+          from: "Dhravya From Supermemory <waitlist@m.supermemory.com>",
+          to: email,
+          subject: "You're in the waitlist - A personal note from Dhravya",
+          html: `<p>Hi. I'm Dhravya. I'm building Supermemory to help people remember everything.<br></br> ${message} <br></br><br></br>I'll be in touch when we launch! Till then, just reply to this email if you wanna talk :)<br></br>If you want to follow me on X, here's my handle: <a href='https://x.com/dhravyashah'>@dhravyashah</a><br></br><br></br>- Dhravya</p>`,
+        });
+      } catch (e) {
+        console.error(e);
+        return c.json({ error: "Failed to add to waitlist" }, 400);
+      }
+
+      return c.json({ success: true });
+    }
+  )
+  .onError((err, c) => {
+    console.error(err);
+    return c.json({ error: "Internal server error" }, 500);
+  });
+
+export default {
+  fetch: app.fetch,
+};
+
+export { ContentWorkflow, DurableObjectRateLimiter };
+
+export type AppType = typeof app;

apps/backend/src/providers.ts

@@ -0,0 +1,19 @@
+import { createOpenAI, OpenAIProvider } from "@ai-sdk/openai";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { Env } from "./types";
+
+export function openai(
+  env: Env,
+  apiKey?: string
+): ReturnType<typeof createOpenAI> {
+  return createOpenAI({
+    apiKey: apiKey || env.OPEN_AI_API_KEY,
+    baseURL: "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai"
+  });
+}
+
+export function google(securityKey: string) {
+  return createGoogleGenerativeAI({
+    apiKey: securityKey,
+  });
+}

apps/backend/src/routes/actions.ts

@@ -0,0 +1,861 @@
+import { Hono } from "hono";
+import { Variables, Env, recommendedQuestionsSchema } from "../types";
+import { zValidator } from "@hono/zod-validator";
+import { z } from "zod";
+import {
+  AISDKError,
+  convertToCoreMessages,
+  embed,
+  generateObject,
+  InvalidPromptError,
+  Message,
+  smoothStream,
+  StreamData,
+  streamText,
+  TextPart,
+} from "ai";
+import {
+  chatThreads,
+  documents,
+  chunk,
+  spaces,
+  spaceAccess,
+} from "@supermemory/db/schema";
+import { google, openai } from "../providers";
+import { randomId } from "@supermemory/shared";
+import { and, cosineDistance, database, desc, eq, or, sql } from "@supermemory/db";
+import { typeDecider } from "../utils/typeDecider";
+import { isErr, Ok } from "../errors/results";
+
+const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
+  .post(
+    "/chat",
+    zValidator(
+      "json",
+      z.object({
+        messages: z.array(z.any()).min(1, "At least one message is required"),
+        threadId: z.string().optional(),
+      })
+    ),
+    async (c) => {
+      const user = c.get("user");
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const { messages, threadId } = await c.req.valid("json");
+
+      // TODO: add rate limiting
+
+      const unfilteredCoreMessages = convertToCoreMessages(
+        (messages as Message[])
+          .filter((m) => m.content.length > 0)
+          .map((m) => ({
+            ...m,
+            content:
+              m.content +
+              (m.annotations
+                ? `<context>${JSON.stringify(m.annotations)}</context>`
+                : ""),
+            experimental_attachments:
+              m.experimental_attachments &&
+              m.experimental_attachments.length > 0
+                ? m.experimental_attachments
+                : (m.data as { files: [] })?.files,
+          }))
+      );
+
+      // make sure that there is no empty messages. if there is, remove it.
+      const coreMessages = unfilteredCoreMessages.filter(
+        (message) => message.content.length > 0
+      );
+      // .map(async (c) => {
+      //   if (
+      //     Array.isArray(c.content) &&
+      //     c.content.some((c) => c.type !== "text")
+      //   ) {
+      //     // convert attachments (IMAGE and files) to base64 by fetching them
+      //     const attachments = c.content.filter((c) => c.type !== "text");
+      //     const base64Attachments = await Promise.all(
+      //       attachments.map(async (a) => {
+      //         const type = (a as ImagePart | FilePart).type;
+      //         if (type === "image") {
+      //           const response = await fetch((a as ImagePart).image.toString());
+      //           return response.arrayBuffer();
+      //         } else if (type === "file") {
+      //           const response = await fetch((a as FilePart).data.toString());
+      //           return response.arrayBuffer();
+      //         }
+      //       })
+      //     );
+      //   }
+      // });
+
+      console.log("Core messages", JSON.stringify(coreMessages, null, 2));
+
+      let threadUuid = threadId;
+
+      const { initLogger, wrapAISDKModel } = await import("braintrust");
+
+      const logger = initLogger({
+        projectName: "supermemory",
+        apiKey: c.env.BRAINTRUST_API_KEY,
+      });
+
+      // const gemini = createOpenAI({
+      //   apiKey: c.env.GEMINI_API_KEY,
+      //   baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
+      // });
+      const openaiClient = openai(c.env);
+
+      const googleClient = wrapAISDKModel(
+        // google(c.env.GEMINI_API_KEY).chat("gemini-exp-1206")
+        openai(c.env).chat("gpt-4o")
+      );
+
+      // Create new thread if none exists
+      if (!threadUuid) {
+        const uuid = randomId();
+        const newThread = await database(c.env.HYPERDRIVE.connectionString)
+          .insert(chatThreads)
+          .values({
+            firstMessage: messages[0].content,
+            userId: user.id,
+            uuid: uuid,
+            messages: coreMessages,
+          })
+          .returning();
+
+        threadUuid = newThread[0].uuid;
+      }
+      const openAi = openai(c.env);
+
+      const model = openAi.embedding("text-embedding-3-large", {
+        dimensions: 1536,
+      });
+
+      let lastUserMessage = coreMessages
+        .reverse()
+        .find((i) => i.role === "user");
+
+      // get the text of lastUserMEssage
+      const queryText =
+        typeof lastUserMessage?.content === "string"
+          ? lastUserMessage.content
+          : lastUserMessage?.content.map((c) => (c as TextPart).text).join("");
+
+      console.log("querytext", queryText);
+
+      const embedStart = performance.now();
+      const { embedding } = await embed({
+        model,
+        value: queryText,
+      });
+      const embedEnd = performance.now();
+      console.log(`Embedding generation took ${embedEnd - embedStart}ms`);
+
+      if (!embedding) {
+        return c.json({ error: "Failed to generate embedding for query" }, 500);
+      }
+
+      // Perform semantic search using cosine similarity
+      // Log the query text to debug what we're searching for
+      console.log("Searching for:", queryText);
+      console.log("user id", user.id);
+
+      const similarity = sql<number>`1 - (${cosineDistance(
+        chunk.embeddings,
+        embedding
+      )})`;
+
+      // Find similar chunks using cosine similarity
+      // Join with documents table to get chunks only from documents the user has access to
+      // First get all results to normalize
+      // Get top 20 results first to avoid processing entire dataset
+      const dbQueryStart = performance.now();
+      const topResults = await database(c.env.HYPERDRIVE.connectionString)
+        .select({
+          similarity,
+          id: documents.id,
+          content: documents.content,
+          type: documents.type,
+          url: documents.url,
+          title: documents.title,
+          createdAt: documents.createdAt,
+          updatedAt: documents.updatedAt,
+          userId: documents.userId,
+          description: documents.description,
+          ogImage: documents.ogImage,
+        })
+        .from(chunk)
+        .innerJoin(documents, eq(chunk.documentId, documents.id))
+        .where(and(eq(documents.userId, user.id), sql`${similarity} > 0.4`))
+        .orderBy(desc(similarity));
+
+      // Get unique documents with their highest similarity chunks
+      const uniqueDocuments = Object.values(
+        topResults.reduce(
+          (acc, curr) => {
+            if (
+              !acc[curr.id] ||
+              acc[curr.id].content === curr.content ||
+              acc[curr.id].url === curr.url
+            ) {
+              acc[curr.id] = curr;
+            }
+            return acc;
+          },
+          {} as Record<number, (typeof topResults)[0]>
+        )
+      ).slice(0, 5);
+
+      const dbQueryEnd = performance.now();
+      console.log(`Database query took ${dbQueryEnd - dbQueryStart}ms`);
+
+      // Calculate min/max once for the subset
+      const processingStart = performance.now();
+      const minSimilarity = Math.min(
+        ...uniqueDocuments.map((r) => r.similarity)
+      );
+      const maxSimilarity = Math.max(
+        ...uniqueDocuments.map((r) => r.similarity)
+      );
+      const range = maxSimilarity - minSimilarity;
+
+      // Normalize the results
+      const normalizedResults = uniqueDocuments.map((result) => ({
+        ...result,
+        normalizedSimilarity:
+          range === 0 ? 1 : (result.similarity - minSimilarity) / range,
+      }));
+
+      // Get either all results above 0.6 threshold, or at least top 3 results
+      const results = normalizedResults
+        .sort((a, b) => b.normalizedSimilarity - a.normalizedSimilarity)
+        .slice(
+          0,
+          Math.max(
+            3,
+            normalizedResults.filter((r) => r.normalizedSimilarity > 0.6).length
+          )
+        );
+
+      const processingEnd = performance.now();
+      console.log(
+        `Results processing took ${processingEnd - processingStart}ms`
+      );
+
+      const cleanDocumentsForContext = results.map((d) => ({
+        title: d.title,
+        description: d.description,
+        url: d.url,
+        type: d.type,
+        content: d.content,
+      }));
+
+      // Update lastUserMessage with search results
+      const messageUpdateStart = performance.now();
+      if (lastUserMessage) {
+        lastUserMessage.content =
+          typeof lastUserMessage.content === "string"
+            ? lastUserMessage.content +
+              `<context>${JSON.stringify(cleanDocumentsForContext)}</context>`
+            : [
+                ...lastUserMessage.content,
+                {
+                  type: "text",
+                  text: `<context>${JSON.stringify(cleanDocumentsForContext)}</context>`,
+                },
+              ];
+      }
+
+      // edit the last coreusermessage in the array
+      if (lastUserMessage) {
+        coreMessages[coreMessages.length - 1] = lastUserMessage;
+      }
+      const messageUpdateEnd = performance.now();
+      console.log(
+        `Message update took ${messageUpdateEnd - messageUpdateStart}ms`
+      );
+
+      try {
+        const streamStart = performance.now();
+        const result = await streamText({
+          model: googleClient,
+          experimental_providerMetadata: {
+            metadata: {
+              userId: user.id,
+              chatThreadId: threadUuid,
+            },
+          },
+          experimental_transform: smoothStream(),
+          messages: [
+            {
+              role: "system",
+              content: `You are a knowledgeable and helpful AI assistant for Supermemory, a personal knowledge management app. Your goal is to help users explore and understand their saved content.
+  
+  Key guidelines:
+  - Maintain natural, engaging conversation while seamlessly incorporating relevant information from the user's knowledge base
+  - Build on previous messages in the conversation to provide coherent, contextual responses
+  - Be concise but thorough, focusing on the most relevant details
+  - When appropriate, make connections between different pieces of information
+  - If you're not sure about something, be honest and say so
+  - Feel free to ask clarifying questions if needed
+  - Make it easy to read for the user!
+  - Use markdown to format your responses but dont make your answers TOO long include any and all information related to context in the response if possible.
+  - only talk about the context if the right answer is in the context.
+  - You are Supermemory - a personal knowledge management app.
+  - You are built by Dhravya Shah (https://dhravya.dev). And the supermemory team (https://supermemory.ai).
+  
+  The user's saved content is provided in <context> tags. Use this information naturally without explicitly referencing it.`,
+            },
+            ...coreMessages,
+          ],
+          async onFinish(completion) {
+            try {
+              // remove context from lastUserMessage
+              if (lastUserMessage) {
+                lastUserMessage.content =
+                  typeof lastUserMessage.content === "string"
+                    ? lastUserMessage.content.replace(
+                        /<context>([\s\S]*?)<\/context>/g,
+                        ""
+                      )
+                    : lastUserMessage.content.filter(
+                        (part) =>
+                          !(
+                            part.type === "text" &&
+                            part.text.startsWith("<context>")
+                          )
+                      );
+
+                coreMessages[coreMessages.length - 1] = lastUserMessage;
+              }
+
+              console.log("results", results);
+
+              const newMessages = [
+                ...coreMessages,
+                {
+                  role: "assistant",
+                  content:
+                    completion.text +
+                    `<context>[${JSON.stringify(results)}]</context>`,
+                },
+              ];
+              await data.close();
+
+              if (threadUuid) {
+                await database(c.env.HYPERDRIVE.connectionString)
+                  .update(chatThreads)
+                  .set({ messages: newMessages })
+                  .where(eq(chatThreads.uuid, threadUuid));
+              }
+            } catch (error) {
+              console.error("Failed to update thread:", error);
+              // Continue execution - the message was delivered even if saving failed
+            }
+          },
+        });
+
+        const streamEnd = performance.now();
+        console.log(`Stream response took ${streamEnd - streamStart}ms`);
+
+        const data = new StreamData();
+
+        const context = results.map((r) => ({
+          similarity: r.similarity,
+          id: r.id,
+          content: r.content,
+          type: r.type,
+          url: r.url,
+          title: r.title,
+          description: r.description,
+          ogImage: r.ogImage,
+          userId: r.userId,
+          createdAt: r.createdAt.toISOString(),
+          updatedAt: r.updatedAt?.toISOString() || null,
+        }));
+        // Full context objects in the data
+        data.appendMessageAnnotation(context);
+
+        return result.toDataStreamResponse({
+          headers: {
+            "Supermemory-Thread-Uuid": threadUuid,
+            "Content-Type": "text/x-unknown",
+            "content-encoding": "identity",
+            "transfer-encoding": "chunked",
+          },
+          data,
+        });
+      } catch (error) {
+        console.error("Chat error:", error);
+
+        if (error instanceof InvalidPromptError) {
+          return c.json(
+            { error: "Invalid prompt - please rephrase your message" },
+            400
+          );
+        }
+
+        // Handle database connection errors
+        if ((error as AISDKError).cause === "ECONNREFUSED") {
+          return c.json({ error: "Database connection failed" }, 503);
+        }
+
+        return c.json(
+          {
+            error: "An unexpected error occurred",
+            details:
+              c.env.NODE_ENV === "development"
+                ? (error as Error).message
+                : undefined,
+          },
+          500
+        );
+      }
+    }
+  )
+  .get(
+    "/chat/:threadUuid",
+    zValidator(
+      "param",
+      z.object({
+        threadUuid: z.string(),
+      })
+    ),
+    async (c) => {
+      const user = c.get("user");
+      const threadUuid = c.req.valid("param").threadUuid;
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const chatHistory = await database(c.env.HYPERDRIVE.connectionString)
+        .select()
+        .from(chatThreads)
+        .where(
+          and(eq(chatThreads.userId, user.id), eq(chatThreads.uuid, threadUuid))
+        );
+
+      if (!chatHistory) {
+        return c.json({ error: "Chat history not found" }, 404);
+      }
+
+      return c.json({ chatHistory: chatHistory[0].messages });
+    }
+  )
+  .get("/recommended-questions", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const db = database(c.env.HYPERDRIVE.connectionString);
+    const recentDocuments = await db
+      .select()
+      .from(documents)
+      .where(eq(documents.userId, user.id))
+      .orderBy(sql`RANDOM()`)
+      .limit(3);
+
+    if (recentDocuments.length === 0) {
+      return c.json({ questions: [] });
+    }
+
+    const cachedQuestions = await c.env.MD_CACHE.get(`rq:${user.id}`);
+    if (cachedQuestions) {
+      const randomQuestions = JSON.parse(cachedQuestions)
+        .questions.sort(() => Math.random() - 0.5)
+        .slice(0, 3);
+      return c.json({ questions: randomQuestions });
+    }
+
+    const { initLogger, wrapAISDKModel } = await import("braintrust");
+
+    const logger = initLogger({
+      projectName: "supermemory",
+      apiKey: c.env.BRAINTRUST_API_KEY,
+    });
+
+    const model = wrapAISDKModel(openai(c.env).chat("gpt-4o-mini-2024-07-18"));
+
+    const aiResponse = await generateObject({
+      schema: z.object({
+        questions: recommendedQuestionsSchema,
+      }),
+      model,
+      prompt: `You are helping generate search suggestions for a user's personal knowledge base. 
+      
+  Generate 10 specific, focused questions based on the following documents. The questions should:
+  - Be highly specific and reference concrete details from the documents
+  - Focus on key insights, important facts, or interesting relationships
+  - Be phrased naturally, as if the user is trying to recall something they learned
+  - Be 2-8 words long
+  - Not include generic questions that could apply to any document
+  
+  Documents:
+  ${recentDocuments.map((d) => d.content).join("\n\n")}`,
+    });
+
+    await c.env.MD_CACHE.put(
+      `rq:${user.id}`,
+      JSON.stringify(aiResponse.object),
+      {
+        // 3 hours
+        expirationTtl: 10800,
+      }
+    );
+
+    const questions = aiResponse.object.questions;
+    const randomQuestions = questions
+      .sort(() => Math.random() - 0.5)
+      .slice(0, 3);
+    return c.json({ questions: randomQuestions });
+  })
+  .get("/suggested-learnings", async (c) => {
+    const user = await c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const db = database(c.env.HYPERDRIVE.connectionString);
+
+    // Try to get from cache first
+    const cacheKey = `sl:${user.id}`;
+    const cached = await c.env.MD_CACHE.get(cacheKey);
+    if (cached) {
+      return c.json({
+        suggestedLearnings: JSON.parse(cached) as {[x: string]: string},
+      });
+    }
+
+    // Get random sample of user's documents that are well-distributed
+    const recentLearnings = await db
+      .select()
+      .from(documents)
+      .where(eq(documents.userId, user.id))
+      // Use random() to get distributed sample
+      .orderBy(sql`RANDOM()`)
+      .limit(7);
+
+    if (recentLearnings.length === 0) {
+      return c.json({ suggestedLearnings: [] });
+    }
+
+    // for each document, i want to generate a list of
+    // small markdown tweet-like things that the user might want to remember about
+    const suggestedLearnings = await Promise.all(
+      recentLearnings.map(async (document) => {
+        const model = openai(c.env).chat("gpt-4o-mini-2024-07-18");
+        const prompt = `Generate a concise topic recall card for this document. The card should:
+  - Have a clear title that captures the main topic
+  - Include a brief "Last week, you saved notes on..." intro (do something different every time.)
+  - List 2-3 key points from the content in simple bullet points
+  - Keep the total length under 280 characters
+  - Focus on the core concepts worth remembering
+  - Be in markdown format
+  - if you don't have a good suggestions, just skip that document.
+  
+  Here's the document content: ${document.content}, Document saved at: ${document.updatedAt}, Today's date: ${new Date().toLocaleDateString()}`;
+        const response = await generateObject({
+          schema: z.object({
+            [document.uuid]: z.string(),
+          }),
+          // @ts-ignore
+          model,
+          prompt,
+        });
+        return response.object;
+      })
+    );
+
+    // Cache the results
+    await c.env.MD_CACHE.put(cacheKey, JSON.stringify(suggestedLearnings), {
+      expirationTtl: 60 * 60 * 3, // 3 hours
+    });
+
+    return c.json({ suggestedLearnings });
+  })
+  .post(
+    "/search",
+    zValidator(
+      "json",
+      z.object({
+        query: z.string().min(1, "Search query cannot be empty"),
+        limit: z.number().min(1).max(50).default(10),
+        threshold: z.number().min(0).max(1).default(0),
+      })
+    ),
+    async (c) => {
+      const { query, limit, threshold } = c.req.valid("json");
+      const user = c.get("user");
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const openAi = openai(c.env, c.env.OPEN_AI_API_KEY);
+
+      try {
+        // Generate embedding for the search query
+        const model = openAi.embedding("text-embedding-3-small");
+
+        const embeddings = await embed({ model, value: query });
+
+        if (!embeddings.embedding) {
+          return c.json(
+            { error: "Failed to generate embedding for query" },
+            500
+          );
+        }
+
+        // Perform semantic search using cosine similarity
+        const results = await database(c.env.HYPERDRIVE.connectionString)
+          .select({
+            id: documents.id,
+            uuid: documents.uuid,
+            content: documents.content,
+            createdAt: documents.createdAt,
+            chunkContent: chunk.textContent,
+            similarity: sql<number>`1 - (embeddings <=> ${JSON.stringify(
+              embeddings.embedding
+            )}::vector)`,
+          })
+          .from(chunk)
+          .innerJoin(documents, eq(chunk.documentId, documents.id))
+          .where(
+            and(
+              eq(documents.userId, user.id),
+              sql`1 - (embeddings <=> ${JSON.stringify(embeddings.embedding)}::vector) >= ${threshold}`
+            )
+          )
+          .orderBy(
+            sql`1 - (embeddings <=> ${JSON.stringify(embeddings.embedding)}::vector) desc`
+          ) //figure out a better way to do order by my brain isn't working at this time. but youcan't do vector search twice
+          .limit(limit);
+
+        return c.json({
+          results: results.map((r) => ({
+            ...r,
+            similarity: Number(r.similarity.toFixed(4)),
+          })),
+        });
+      } catch (error) {
+        console.error("[Search Error]", error);
+        return c.json(
+          {
+            error: "Search failed",
+            details:
+              c.env.NODE_ENV === "development"
+                ? (error as Error).message
+                : undefined,
+          },
+          500
+        );
+      }
+    }
+  )
+  .post(
+    "/add",
+    zValidator(
+      "json",
+      z.object({
+        content: z.string().min(1, "Content cannot be empty"),
+        spaces: z.array(z.string()).max(5).optional(),
+        prefetched: z
+          .object({
+            contentToVectorize: z.string(),
+            contentToSave: z.string(),
+            title: z.string(),
+            type: z.string(),
+            description: z.string().optional(),
+            ogImage: z.string().optional(),
+          })
+          .optional(),
+      })
+    ),
+    async (c) => {
+      const body = c.req.valid("json");
+      console.log("body", body);
+      const user = c.get("user");
+
+      if (!user) {
+        return c.json({ error: "You must be logged in to add content" }, 401);
+      }
+
+      const type = body.prefetched
+        ? Ok(body.prefetched.type)
+        : typeDecider(body.content);
+
+      if (isErr(type)) {
+        return c.json(
+          {
+            error: "Could not determine content type",
+            details: type.error.message,
+          },
+          400
+        );
+      }
+
+      if (type.value === "page" && !body.content.startsWith("http")) {
+        body.content = `https://${body.content}`;
+      }
+
+      const uuid = randomId();
+      const contentId = `add-${user.id}-${uuid}`;
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      // Calculate document hash early to enable faster duplicate detection
+      const content = body.prefetched?.contentToVectorize || body.content;
+      const encoder = new TextEncoder();
+      const data = encoder.encode(content);
+      const hashBuffer = await crypto.subtle.digest("SHA-256", data);
+      const hashArray = Array.from(new Uint8Array(hashBuffer));
+      const documentHash = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
+
+      // Check for duplicates using hash
+      const existingDocs = await db
+        .select()
+        .from(documents)
+        .where(
+          and(
+            eq(documents.userId, user.id),
+            or(
+              eq(documents.contentHash, documentHash),
+              and(
+                eq(documents.type, type.value),
+                or(
+                  eq(documents.url, body.content),
+                  eq(documents.raw, content)
+                )
+              )
+            )
+          )
+        );
+
+      if (existingDocs.length > 0) {
+        return c.json(
+          { error: `That ${type.value} already exists in your memories` },
+          409
+        );
+      }
+
+      // Check space permissions if spaces are specified
+      if (body.spaces && body.spaces.length > 0) {
+        const spacePermissions = await Promise.all(
+          body.spaces.map(async (spaceId) => {
+            const space = await db
+              .select()
+              .from(spaces)
+              .where(eq(spaces.uuid, spaceId))
+              .limit(1);
+
+            if (!space[0]) {
+              return { spaceId, allowed: false, error: "Space not found" };
+            }
+
+            // If public space, only owner can add content
+            if (space[0].isPublic) {
+              return {
+                spaceId,
+                allowed: space[0].ownerId === user.id,
+                error:
+                  space[0].ownerId !== user.id
+                    ? "Only space owner can add to public spaces"
+                    : null,
+              };
+            }
+
+            // For private spaces, check if user is owner or in allowlist
+            const spaceAccessCheck = await db
+              .select()
+              .from(spaceAccess)
+              .where(
+                and(
+                  eq(spaceAccess.spaceId, space[0].id),
+                  eq(spaceAccess.userEmail, user.email),
+                  eq(spaceAccess.status, "accepted")
+                )
+              )
+              .limit(1);
+
+            return {
+              spaceId,
+              allowed:
+                space[0].ownerId === user.id || spaceAccessCheck.length > 0,
+              error:
+                space[0].ownerId !== user.id && !spaceAccessCheck.length
+                  ? "Not authorized to add to this space"
+                  : null,
+            };
+          })
+        );
+
+        const unauthorized = spacePermissions.filter((p) => !p.allowed);
+        if (unauthorized.length > 0) {
+          return c.json(
+            {
+              error: "Space permission denied",
+              details: unauthorized
+                .map((u) => `${u.spaceId}: ${u.error}`)
+                .join(", "),
+            },
+            403
+          );
+        }
+      }
+
+      const isExternalContent = [
+        "page",
+        "tweet",
+        "document",
+        "notion",
+      ].includes(type.value);
+      const indexedUrl = isExternalContent
+        ? body.content
+        : `https://supermemory.ai/content/${contentId}`;
+
+      // Insert into documents table with hash
+      try {
+        await db.insert(documents).values({
+          uuid: contentId,
+          userId: user.id,
+          type: type.value,
+          url: indexedUrl,
+          title: body.prefetched?.title,
+          description: body.prefetched?.description,
+          ogImage: body.prefetched?.ogImage,
+          contentHash: documentHash,
+          raw:
+            (body.prefetched ?? body.content) + "\n\n" + body.spaces?.join(" "),
+        });
+
+        await c.env.CONTENT_WORKFLOW.create({
+          params: {
+            userId: user.id,
+            content: body.content,
+            spaces: body.spaces,
+            type: type.value,
+            uuid: contentId,
+            url: indexedUrl,
+            prefetched: body.prefetched,
+          },
+          id: contentId,
+        });
+
+        return c.json({
+          message: "Content added successfully",
+          id: contentId,
+          type: type.value,
+        });
+      } catch (error) {
+        console.error("[Add Content Error]", error);
+        return c.json({ error: "Failed to process content" }, 500);
+      }
+    }
+  );
+
+export default actions;

apps/backend/src/routes/integrations.ts

@@ -0,0 +1,177 @@
+import { Hono } from "hono";
+import { Env, Variables } from "../types";
+import { getDecryptedKV } from "encrypt-workers-kv";
+import { getAllNotionPageContents } from "../utils/notion";
+import { and, eq, or } from "@supermemory/db";
+import { documents } from "@supermemory/db/schema";
+import { database } from "@supermemory/db";
+
+const integrations = new Hono<{ Variables: Variables; Bindings: Env }>().get("/notion/import", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    // Create SSE stream
+    const stream = new TransformStream();
+    const writer = stream.writable.getWriter();
+    const encoder = new TextEncoder();
+
+    // Create response first so client gets headers immediately
+    const response = new Response(stream.readable, {
+      headers: {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        Connection: "keep-alive",
+        // Required CORS headers for SSE
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Credentials": "true",
+      },
+    });
+
+    const sendMessage = async (data: Record<string, any>) => {
+      // Proper SSE format requires "data: " prefix and double newline
+      const formattedData = `data: ${JSON.stringify(data)}\n\n`;
+      await writer.write(encoder.encode(formattedData));
+    };
+
+    // Start processing in background
+    c.executionCtx.waitUntil(
+      (async () => {
+        try {
+          // Send initial heartbeat
+          await sendMessage({ type: "connected" });
+
+          const token = await getDecryptedKV(
+            c.env.ENCRYPTED_TOKENS,
+            `${user.uuid}-notion`,
+            `${c.env.WORKOS_COOKIE_PASSWORD}-${user.uuid}`
+          );
+
+          const stringToken = new TextDecoder().decode(token);
+          if (!stringToken) {
+            await sendMessage({ type: "error", error: "No token found" });
+            await writer.close();
+            return;
+          }
+
+          await sendMessage({ type: "progress", progress: 5 });
+
+          // Fetch pages with progress updates
+          const pages = await getAllNotionPageContents(
+            stringToken,
+            async (progress) => {
+              // Map progress from 0-100 to 5-40 range
+              const scaledProgress = Math.floor(5 + (progress * 35) / 100);
+              await sendMessage({ type: "progress", progress: scaledProgress });
+            }
+          );
+
+          await sendMessage({ type: "progress", progress: 40 });
+
+          let processed = 0;
+          const totalPages = pages.length;
+
+          const db = database(c.env.HYPERDRIVE.connectionString);
+
+          for (const page of pages) {
+            // Calculate document hash for duplicate detection
+            const encoder = new TextEncoder();
+            const data = encoder.encode(page.content);
+            const hashBuffer = await crypto.subtle.digest("SHA-256", data);
+            const hashArray = Array.from(new Uint8Array(hashBuffer));
+            const documentHash = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
+
+            // Check for duplicates using hash
+            const existingDocs = await db
+              .select()
+              .from(documents)
+              .where(
+                and(
+                  eq(documents.userId, user.id),
+                  or(
+                    eq(documents.contentHash, documentHash),
+                    and(
+                      eq(documents.type, "notion"),
+                      or(
+                        eq(documents.url, page.url),
+                        eq(documents.raw, page.content)
+                      )
+                    )
+                  )
+                )
+              );
+
+            if (existingDocs.length > 0) {
+              await sendMessage({ 
+                type: "warning", 
+                message: `Skipping duplicate page: ${page.title}` 
+              });
+              processed++;
+              continue;
+            }
+
+            // Insert into documents table first
+            try {
+              await db.insert(documents).values({
+                uuid: page.id,
+                userId: user.id,
+                type: "notion",
+                url: page.url,
+                title: page.title,
+                contentHash: documentHash,
+                raw: page.content,
+              });
+
+              await c.env.CONTENT_WORKFLOW.create({
+                params: {
+                  userId: user.id,
+                  content: page.url,
+                  spaces: [],
+                  type: "notion",
+                  uuid: page.id,
+                  url: page.url,
+                  prefetched: {
+                    contentToVectorize: page.content,
+                    contentToSave: page.content,
+                    title: page.title,
+                    type: "notion",
+                  },
+                  createdAt: page.createdAt,
+                },
+                id: `${user.id}-${page.id}-${new Date().getTime()}`,
+              });
+
+              processed++;
+              const progress = 50 + Math.floor((processed / totalPages) * 50);
+              await sendMessage({ type: "progress", progress, page: page.title });
+
+            } catch (error) {
+              console.error(`Failed to process page ${page.title}:`, error);
+              await sendMessage({
+                type: "warning",
+                message: `Failed to process page: ${page.title}`,
+                error: error instanceof Error ? error.message : "Unknown error"
+              });
+              processed++;
+              continue;
+            }
+          }
+
+          await sendMessage({ type: "complete", progress: 100 });
+          await writer.close();
+        } catch (error) {
+          console.error("Import error:", error);
+          await sendMessage({
+            type: "error",
+            error: error instanceof Error ? error.message : "Import failed",
+          });
+          await writer.close();
+        }
+      })()
+    );
+
+    return response;
+  });
+
+export default integrations;

apps/backend/src/routes/memories.ts

@@ -0,0 +1,203 @@
+import { Hono } from "hono";
+import { Variables, Env } from "../types";
+import { zValidator } from "@hono/zod-validator";
+import { z } from "zod";
+import {
+  documents,
+  spaces,
+  spaceAccess,
+  contentToSpace,
+} from "@supermemory/db/schema";
+import { and, database, desc, eq, sql } from "@supermemory/db";
+
+
+const memories = new Hono<{ Variables: Variables; Bindings: Env }>()
+  .get(
+    "/",
+    zValidator(
+      "query",
+      z.object({
+        start: z.string().default("0").transform(Number),
+        count: z.string().default("10").transform(Number),
+        spaceId: z.string().optional(),
+      })
+    ),
+    async (c) => {
+      const { start, count, spaceId } = c.req.valid("query");
+      const user = c.get("user");
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      console.log("Fetching memories with spaceId", spaceId);
+      console.log(c.req.url);
+      // If spaceId provided, verify access
+      if (spaceId) {
+        console.log("SpaceID provided", spaceId);
+        const space = await db
+          .select()
+          .from(spaces)
+          .where(eq(spaces.uuid, spaceId.split("---")[0]))
+          .limit(1);
+
+        if (!space[0]) {
+          return c.json({ error: "Space not found" }, 404);
+        }
+
+        // Check access - allow if public, user owns the space, or has access through spaceAccess
+        if (!space[0].isPublic && !user) {
+          return c.json({ error: "Unauthorized" }, 401);
+        }
+
+        if (!space[0].isPublic && space[0].ownerId !== user?.id) {
+          const access = await db
+            .select()
+            .from(spaceAccess)
+            .where(
+              and(
+                eq(spaceAccess.spaceId, space[0].id),
+                eq(spaceAccess.userEmail, user?.email ?? ""),
+                eq(spaceAccess.status, "accepted")
+              )
+            )
+            .limit(1);
+
+          if (access.length === 0) {
+            console.log("Unauthorized access to", c.req.url);
+            return c.json({ error: "Unauthorized" }, 401);
+          }
+        }
+
+        // Get documents for space
+        const [items, totalResult] = await Promise.all([
+          db
+            .select({
+              documents,
+            })
+            .from(documents)
+            .innerJoin(
+              contentToSpace,
+              eq(documents.id, contentToSpace.contentId)
+            )
+            .where(eq(contentToSpace.spaceId, space[0].id))
+            .orderBy(desc(documents.createdAt))
+            .limit(count)
+            .offset(start),
+          db
+            .select({
+              total: sql<number>`count(*)`.as("total"),
+            })
+            .from(documents)
+            .innerJoin(
+              contentToSpace,
+              eq(documents.id, contentToSpace.contentId)
+            )
+            .where(eq(contentToSpace.spaceId, space[0].id)),
+        ]);
+
+        const total = totalResult[0]?.total ?? 0;
+
+        return c.json({
+          items: items.map((item) => ({
+            ...item.documents,
+            id: item.documents.uuid,
+          })),
+          total,
+        });
+      }
+
+      // Regular user memories endpoint
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      // Set cache control headers for 5 minutes
+      c.header("Cache-Control", "private, max-age=300");
+      c.header("Vary", "Cookie"); // Vary on Cookie since response depends on user
+
+      // Generate ETag based on user ID, start, count
+      const etag = `"${user.id}-${start}-${count}"`;
+      c.header("ETag", etag);
+
+      // Check if client has matching ETag
+      const ifNoneMatch = c.req.header("If-None-Match");
+      if (ifNoneMatch === etag) {
+        return new Response(null, { status: 304 });
+      }
+
+      const [items, [{ total }]] = await Promise.all([
+        db
+          .select()
+          .from(documents)
+          .where(eq(documents.userId, user.id))
+          .orderBy(desc(documents.createdAt))
+          .limit(count)
+          .offset(start),
+        db
+          .select({
+            total: sql<number>`count(*)`.as("total"),
+          })
+          .from(documents)
+          .where(eq(documents.userId, user.id)),
+      ]);
+
+      return c.json({
+        items: items.map((item) => ({
+          ...item,
+          id: item.uuid,
+        })),
+        total,
+      });
+    }
+  )
+  .get(
+    "/:id",
+    zValidator("param", z.object({ id: z.string() })),
+    async (c) => {
+      const { id } = c.req.valid("param");
+      const user = c.get("user");
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const memory = await database(c.env.HYPERDRIVE.connectionString)
+        .select()
+        .from(documents)
+        .where(and(eq(documents.uuid, id), eq(documents.userId, user.id)))
+        .limit(1);
+
+      return c.json(memory[0]);
+    }
+  )
+  .delete(
+    "/:id",
+    zValidator("param", z.object({ id: z.string() })),
+    async (c) => {
+      const { id } = c.req.valid("param");
+      const user = c.get("user");
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      const doc = await db
+        .select()
+        .from(documents)
+        .where(and(eq(documents.uuid, id), eq(documents.userId, user.id)))
+        .limit(1);
+
+      if (!doc[0]) {
+        return c.json({ error: "Document not found" }, 404);
+      }
+
+      const [document, contentToSpacei] = await Promise.all([
+        db.delete(documents).where(and(eq(documents.uuid, id), eq(documents.userId, user.id))),
+        db.delete(contentToSpace).where(eq(contentToSpace.contentId, doc[0].id)),
+      ]);
+
+      return c.json({ success: true });
+    }
+  );
+
+export default memories;

apps/backend/src/routes/spaces.ts

@@ -0,0 +1,529 @@
+import { Hono } from "hono";
+import { Env, Variables } from "../types";
+import { and, database, desc, eq, isNotNull, or, sql } from "@supermemory/db";
+import {
+  contentToSpace,
+  documents,
+  savedSpaces,
+  spaceAccess,
+  spaces,
+  users,
+} from "@supermemory/db/schema";
+import { zValidator } from "@hono/zod-validator";
+import { z } from "zod";
+import { randomId } from "@supermemory/shared";
+
+const spacesRoute = new Hono<{ Variables: Variables; Bindings: Env }>()
+  .get("/", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const db = database(c.env.HYPERDRIVE.connectionString);
+
+    const [allSpaces, savedSpacesList, spaceOwners] = await Promise.all([
+      db
+        .select({
+          id: spaces.id,
+          uuid: spaces.uuid,
+          name: sql<string>`REGEXP_REPLACE(${spaces.name}, E'[\\n\\r]+', ' ', 'g')`.as(
+            "name"
+          ),
+          ownerId: spaces.ownerId,
+          isPublic: spaces.isPublic,
+          createdAt: spaces.createdAt,
+          accessType: spaceAccess.accessType,
+        })
+        .from(spaces)
+        .leftJoin(
+          spaceAccess,
+          and(
+            eq(spaces.id, spaceAccess.spaceId),
+            eq(spaceAccess.userEmail, user.email),
+            eq(spaceAccess.status, "accepted")
+          )
+        )
+        .where(or(eq(spaces.ownerId, user.id), isNotNull(spaceAccess.spaceId)))
+        .orderBy(desc(spaces.createdAt)),
+
+      db
+        .select({
+          spaceId: savedSpaces.spaceId,
+        })
+        .from(savedSpaces)
+        .where(eq(savedSpaces.userId, user.id)),
+
+      db
+        .select({
+          id: users.id,
+          uuid: users.uuid,
+          name: users.firstName,
+          email: users.email,
+          profileImage: users.profilePictureUrl,
+        })
+        .from(users)
+        .innerJoin(spaces, eq(spaces.ownerId, users.id)),
+    ]);
+
+    const savedSpaceIds = new Set(savedSpacesList.map((s) => s.spaceId));
+    const ownerMap = new Map(spaceOwners.map((owner) => [owner.id, owner]));
+
+    const spacesWithDetails = allSpaces.map((space) => {
+      const isOwner = space.ownerId === user.id;
+      const owner = ownerMap.get(space.ownerId);
+
+      return {
+        ...space,
+        favorited: savedSpaceIds.has(space.id),
+        permissions: {
+          canRead: space.isPublic || isOwner || space.accessType != null,
+          canEdit: isOwner || space.accessType === "edit",
+          isOwner,
+        },
+        owner: isOwner
+          ? null
+          : {
+              id: owner?.uuid,
+              name: owner?.name,
+              email: owner?.email,
+              profileImage: owner?.profileImage,
+            },
+      };
+    });
+
+    return c.json({ spaces: spacesWithDetails });
+  }).get("/:spaceId", async (c) => {
+    const user = c.get("user");
+    const spaceId = c.req.param("spaceId");
+    const db = database(c.env.HYPERDRIVE.connectionString);
+  
+    const space = await db
+      .select()
+      .from(spaces)
+      .where(eq(spaces.uuid, spaceId))
+      .limit(1);
+  
+    if (!space[0]) {
+      return c.json({ error: "Space not found" }, 404);
+    }
+  
+    // For public spaces, anyone can read but only owner can edit
+    if (space[0].isPublic) {
+      const canEdit = user?.id === space[0].ownerId;
+      return c.json({
+        ...space[0],
+        permissions: {
+          canRead: true,
+          canEdit,
+          isOwner: space[0].ownerId === user?.id,
+        },
+      });
+    }
+  
+    // For private spaces, require authentication
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+  
+    // Check if user is owner or has access via spaceAccess
+    const isOwner = space[0].ownerId === user.id;
+    let canEdit = isOwner;
+    if (!isOwner) {
+      const spaceAccessCheck = await db
+        .select()
+        .from(spaceAccess)
+        .where(
+          and(
+            eq(spaceAccess.spaceId, space[0].id),
+            eq(spaceAccess.userEmail, user.email),
+            eq(spaceAccess.status, "accepted")
+          )
+        )
+        .limit(1);
+  
+      if (spaceAccessCheck.length === 0) {
+        return c.json({ error: "Access denied" }, 403);
+      }
+  
+      canEdit = spaceAccessCheck[0].accessType === "edit";
+    }
+  
+    return c.json({
+      ...space[0],
+      permissions: {
+        canRead: true,
+        canEdit,
+        isOwner: space[0].ownerId === user.id,
+      },
+    });
+  })
+  .post(
+    "/create",
+    zValidator(
+      "json",
+      z.object({
+        spaceName: z.string().min(1, "Space name cannot be empty").max(100),
+        isPublic: z.boolean(), // keep this explicit please
+      })
+    ),
+    async (c) => {
+      const body = c.req.valid("json");
+      const user = c.get("user");
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+      const uuid = randomId();
+
+      try {
+        const space = await db
+          .insert(spaces)
+          .values({
+            name: body.spaceName.trim(),
+            ownerId: user.id,
+            uuid,
+            isPublic: body.isPublic,
+            createdAt: new Date(),
+          })
+          .returning();
+
+        return c.json({
+          message: "Space created successfully",
+          space: {
+            uuid: space[0].uuid,
+            name: space[0].name,
+            ownerId: space[0].ownerId,
+            isPublic: space[0].isPublic,
+            createdAt: space[0].createdAt,
+          },
+        });
+      } catch (error) {
+        console.error("[Space Creation Error]", error);
+        return c.json({ error: "Failed to create space" }, 500);
+      }
+    }
+  )
+  .post(
+    "/:spaceId/favorite",
+    zValidator(
+      "param",
+      z.object({
+        spaceId: z.string(),
+      })
+    ),
+    async (c) => {
+      const user = c.get("user");
+      const { spaceId } = c.req.valid("param");
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      // Get space details
+      const space = await db
+        .select()
+        .from(spaces)
+        .where(eq(spaces.uuid, spaceId))
+        .limit(1);
+
+      if (!space[0]) {
+        return c.json({ error: "Space not found" }, 404);
+      }
+
+      // Check if it's user's own space
+      if (space[0].ownerId === user.id) {
+        return c.json({ error: "Cannot favorite your own space" }, 400);
+      }
+
+      try {
+        await db.insert(savedSpaces).values({
+          userId: user.id,
+          spaceId: space[0].id,
+          savedAt: new Date(),
+        });
+
+        return c.json({ message: "Space favorited successfully" });
+      } catch (error) {
+        if (
+          error instanceof Error &&
+          error.message.includes("saved_spaces_user_space_idx")
+        ) {
+          // Space is already favorited - remove it
+          await db
+            .delete(savedSpaces)
+            .where(
+              and(
+                eq(savedSpaces.userId, user.id),
+                eq(savedSpaces.spaceId, space[0].id)
+              )
+            );
+          return c.json({ message: "Space unfavorited successfully" });
+        }
+        throw error;
+      }
+    }
+  )
+  .post(
+    "/addContent",
+    zValidator(
+      "json",
+      z.object({
+        spaceId: z.string(),
+        documentId: z.string(),
+      })
+    ),
+    async (c) => {
+      const body = c.req.valid("json");
+      const user = c.get("user");
+      const { spaceId, documentId } = body;
+
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      try {
+        await db.transaction(async (tx) => {
+          // Get space and document, verify space ownership
+          const results = (
+            await tx
+              .select({
+                spaceId: spaces.id,
+                documentId: documents.id,
+                ownerId: spaces.ownerId,
+              })
+              .from(spaces)
+              .innerJoin(
+                documents,
+                and(eq(spaces.uuid, spaceId), eq(documents.uuid, documentId))
+              )
+              .limit(1)
+          )[0];
+
+          if (!results) {
+            return c.json({ error: "Space or document not found" }, 404);
+          }
+
+          if (results.ownerId !== user.id) {
+            return c.json(
+              { error: "Not authorized to modify this space" },
+              403
+            );
+          }
+
+          // Check if mapping already exists to avoid duplicates
+          const existing = await tx
+            .select()
+            .from(contentToSpace)
+            .where(
+              and(
+                eq(contentToSpace.contentId, results.documentId),
+                eq(contentToSpace.spaceId, results.spaceId)
+              )
+            )
+            .limit(1);
+
+          if (existing.length > 0) {
+            return c.json({ error: "Content already exists in space" }, 409);
+          }
+
+          await tx
+            .insert(contentToSpace)
+            .values({
+              contentId: results.documentId,
+              spaceId: results.spaceId,
+            });
+        });
+
+        return c.json({ success: true });
+      } catch (e) {
+        console.error("Failed to add content to space:", e);
+        return c.json(
+          {
+            error: "Failed to add content to space",
+            details: e instanceof Error ? e.message : "Unknown error",
+          },
+          500
+        );
+      }
+    }
+  )
+  .post(
+    "/:spaceId/invite",
+    zValidator(
+      "json",
+      z.object({
+        email: z.string().email("Invalid email address"),
+        accessType: z.enum(["read", "edit"], {
+          errorMap: () => ({
+            message: "Access type must be either 'read' or 'edit'",
+          }),
+        }),
+      })
+    ),
+    async (c) => {
+      const user = c.get("user");
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const { spaceId } = c.req.param();
+      const { email, accessType } = c.req.valid("json");
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      // Check if space exists and user has permission to invite
+      const space = await db
+        .select()
+        .from(spaces)
+        .where(eq(spaces.uuid, spaceId))
+        .limit(1);
+
+      if (space.length === 0) {
+        return c.json({ error: "Space not found" }, 404);
+      }
+
+      // Only space owner can invite others
+      if (space[0].ownerId !== user.id) {
+        return c.json({ error: "Only space owner can invite users" }, 403);
+      }
+
+      // Check if invite already exists
+      const existingInvite = await db
+        .select()
+        .from(spaceAccess)
+        .where(
+          and(
+            eq(spaceAccess.spaceId, space[0].id),
+            eq(spaceAccess.userEmail, email)
+          )
+        )
+        .limit(1);
+
+      if (existingInvite.length > 0) {
+        return c.json(
+          { error: "User already has access or pending invite" },
+          400
+        );
+      }
+
+      // Create invite
+      await db.insert(spaceAccess).values({
+        spaceId: space[0].id,
+        userEmail: email,
+        accessType,
+        status: "pending",
+      });
+
+      // TODO: send email to the user
+
+      return c.json({ success: true });
+    }
+  )
+  .get(
+    "/:spaceId/invitation",
+    zValidator("param", z.object({ spaceId: z.string() })),
+    async (c) => {
+      const { spaceId } = c.req.valid("param");
+      const user = c.get("user");
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      const db = database(c.env.HYPERDRIVE.connectionString);
+
+      const space = await db
+        .select()
+        .from(spaces)
+        .where(eq(spaces.uuid, spaceId))
+        .limit(1);
+
+      if (space.length === 0) {
+        console.log("Space not found", spaceId);
+        return c.json({ error: "Space not found" }, 401);
+      }
+
+      // Get pending invitation with access type
+      const invitation = await db
+        .select()
+        .from(spaceAccess)
+        .where(
+          and(
+            eq(spaceAccess.spaceId, space[0].id),
+            eq(spaceAccess.userEmail, user.email),
+            eq(spaceAccess.status, "pending")
+          )
+        )
+        .limit(1);
+
+      if (invitation.length === 0) {
+        return c.json({ error: "No pending invitation found" }, 403);
+      }
+
+      return c.json({
+        space: space[0],
+        accessType: invitation[0].accessType,
+      });
+    }
+  ).post(
+    "/api/space/invites/:action",
+    zValidator(
+      "json",
+      z.object({
+        spaceId: z.string().min(5, "Invalid space ID format"),
+      })
+    ),
+    async (c) => {
+      const user = c.get("user");
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+  
+      const db = database(c.env.HYPERDRIVE.connectionString);
+  
+      const { action } = c.req.param();
+      if (action !== "accept" && action !== "reject") {
+        return c.json({ error: "Invalid action" }, 400);
+      }
+  
+      const { spaceId } = c.req.valid("json");
+      console.log("space ID", spaceId);
+  
+      // Get space
+      const space = await db
+        .select()
+        .from(spaces)
+        .where(eq(spaces.uuid, spaceId))
+        .limit(1);
+  
+      if (space.length === 0) {
+        return c.json({ error: "Space not found" }, 404);
+      }
+  
+      // Update invite status
+      const updateResult = await db
+        .update(spaceAccess)
+        .set({ status: action === "accept" ? "accepted" : "rejected" })
+        .where(
+          and(
+            eq(spaceAccess.spaceId, space[0].id),
+            eq(spaceAccess.userEmail, user.email),
+            eq(spaceAccess.status, "pending")
+          )
+        );
+  
+      if (updateResult.length === 0) {
+        return c.json({ error: "No pending invite found" }, 404);
+      }
+  
+      return c.json({ success: true });
+    }
+  );
+
+export default spacesRoute;

apps/backend/src/routes/user.ts

@@ -0,0 +1,199 @@
+import { Hono } from "hono";
+import { Env, Variables } from "../types";
+import { and, database, desc, eq, isNotNull, or, sql } from "@supermemory/db";
+import {
+  chatThreads,
+  savedSpaces,
+  spaceAccess,
+  spaces,
+  users,
+} from "@supermemory/db/schema";
+import { decryptApiKey, getApiKey } from "../auth";
+import { DurableObjectStore } from "@hono-rate-limiter/cloudflare";
+import { rateLimiter } from "hono-rate-limiter";
+
+const user = new Hono<{ Variables: Variables; Bindings: Env }>()
+  .get("/", (c) => {
+    return c.json(c.get("user"));
+  })
+  .get("/spaces", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const db = database(c.env.HYPERDRIVE.connectionString);
+
+    const [allSpaces, savedSpacesList, spaceOwners] = await Promise.all([
+      db
+        .select({
+          id: spaces.id,
+          uuid: spaces.uuid,
+          name: sql<string>`REGEXP_REPLACE(${spaces.name}, E'[\\n\\r]+', ' ', 'g')`.as(
+            "name"
+          ),
+          ownerId: spaces.ownerId,
+          isPublic: spaces.isPublic,
+          createdAt: spaces.createdAt,
+          accessType: spaceAccess.accessType,
+        })
+        .from(spaces)
+        .leftJoin(
+          spaceAccess,
+          and(
+            eq(spaces.id, spaceAccess.spaceId),
+            eq(spaceAccess.userEmail, user.email),
+            eq(spaceAccess.status, "accepted")
+          )
+        )
+        .where(or(eq(spaces.ownerId, user.id), isNotNull(spaceAccess.spaceId)))
+        .orderBy(desc(spaces.createdAt)),
+
+      db
+        .select({
+          spaceId: savedSpaces.spaceId,
+        })
+        .from(savedSpaces)
+        .where(eq(savedSpaces.userId, user.id)),
+
+      db
+        .select({
+          id: users.id,
+          uuid: users.uuid,
+          name: users.firstName,
+          email: users.email,
+          profileImage: users.profilePictureUrl,
+        })
+        .from(users)
+        .innerJoin(spaces, eq(spaces.ownerId, users.id)),
+    ]);
+
+    const savedSpaceIds = new Set(savedSpacesList.map((s) => s.spaceId));
+    const ownerMap = new Map(spaceOwners.map((owner) => [owner.id, owner]));
+
+    const spacesWithDetails = allSpaces.map((space) => {
+      const isOwner = space.ownerId === user.id;
+      const owner = ownerMap.get(space.ownerId);
+
+      return {
+        ...space,
+        favorited: savedSpaceIds.has(space.id),
+        permissions: {
+          canRead: space.isPublic || isOwner || space.accessType != null,
+          canEdit: isOwner || space.accessType === "edit",
+          isOwner,
+        },
+        owner: isOwner
+          ? null
+          : {
+              id: owner?.uuid,
+              name: owner?.name,
+              email: owner?.email,
+              profileImage: owner?.profileImage,
+            },
+      };
+    });
+
+    return c.json({ spaces: spacesWithDetails });
+  })
+  .get("/history", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const history = await database(c.env.HYPERDRIVE.connectionString)
+      .select()
+      .from(chatThreads)
+      .where(eq(chatThreads.userId, user.id))
+      .orderBy(desc(chatThreads.createdAt))
+      .limit(10);
+
+    return c.json({ history });
+  })
+  .get("/invitations", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const db = database(c.env.HYPERDRIVE.connectionString);
+
+    const invitations = await db
+      .select({
+        spaceAccess: spaceAccess,
+        spaceUuid: spaces.uuid,
+        spaceName: spaces.name,
+      })
+      .from(spaceAccess)
+      .innerJoin(spaces, eq(spaceAccess.spaceId, spaces.id))
+      .where(eq(spaceAccess.userEmail, user.email))
+      .limit(100);
+
+    return c.json({ invitations });
+  })
+  .get(
+    "/key",
+    async (c) => {
+      const user = c.get("user");
+      if (!user) {
+        return c.json({ error: "Unauthorized" }, 401);
+      }
+
+      // we need user.id and user.lastApiKeyGeneratedAt
+      const lastApiKeyGeneratedAt = user.lastApiKeyGeneratedAt?.getTime();
+      if (!lastApiKeyGeneratedAt) {
+        return c.json({ error: "No API key generated" }, 400);
+      }
+
+      const key = await getApiKey(
+        user.uuid,
+        lastApiKeyGeneratedAt.toString(),
+        c
+      );
+
+      const decrypted = await decryptApiKey(key, c);
+      return c.json({ key, decrypted });
+    }
+  )
+  .post("/update", async (c) => {
+    const user = c.get("user");
+    if (!user) {
+      return c.json({ error: "Unauthorized" }, 401);
+    }
+
+    const body = await c.req.json();
+
+    // Only allow updating specific safe fields
+    const allowedFields = {
+      firstName: true,
+      lastName: true,
+      profilePictureUrl: true,
+      hasOnboarded: true,
+    };
+
+    const updateData: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(body)) {
+      if (allowedFields[key as keyof typeof allowedFields]) {
+        updateData[key] = value;
+      }
+    }
+
+    if (Object.keys(updateData).length === 0) {
+      return c.json({ error: "No valid fields to update" }, 400);
+    }
+
+    const db = database(c.env.HYPERDRIVE.connectionString);
+
+    await db
+      .update(users)
+      .set({
+        ...updateData,
+        updatedAt: new Date(),
+      })
+      .where(eq(users.id, user.id));
+
+    return c.json({ success: true });
+  })
+
+export default user;

apps/backend/src/types.ts

@@ -0,0 +1,78 @@
+import { DurableObjectRateLimiter } from "@hono-rate-limiter/cloudflare";
+import { Session } from "@supermemory/authkit-remix-cloudflare/src/interfaces";
+import { User } from "@supermemory/db/schema";
+import { z } from "zod";
+
+export type Variables = {
+  user: User | null;
+  session: Session | null;
+};
+
+export type WorkflowParams = {
+  userId: number;
+  content: string;
+  spaces?: string[];
+  type: string;
+  uuid: string;
+  url?: string;
+  prefetched?: {
+    contentToVectorize: string;
+    contentToSave: string;
+    title: string;
+    type: string;
+    description: string;
+    ogImage: string;
+  };
+  createdAt: string;
+};
+
+export type Env = {
+  WORKOS_API_KEY: string;
+  WORKOS_CLIENT_ID: string;
+  WORKOS_COOKIE_PASSWORD: string;
+  DATABASE_URL: string;
+  CONTENT_WORKFLOW: Workflow;
+  GEMINI_API_KEY: string;
+  NODE_ENV: string;
+  OPEN_AI_API_KEY: string;
+  BRAINTRUST_API_KEY: string;
+  RESEND_API_KEY: string;
+  TURNSTILE_SECRET_KEY: string;
+
+  MD_CACHE: KVNamespace;
+  HYPERDRIVE: Hyperdrive;
+  EMAIL_LIMITER: {
+    limit: (params: { key: string }) => Promise<{ success: boolean }>;
+  };
+  ENCRYPTED_TOKENS: KVNamespace;
+  RATE_LIMITER: DurableObjectNamespace<DurableObjectRateLimiter>;
+};
+
+export type JobData = {
+  content: string;
+  spaces?: Array<string>;
+  user: number;
+  type: string;
+};
+
+type BaseChunks = {
+  type: "tweet" | "page" | "note" | "image";
+};
+
+export type PageOrNoteChunks = BaseChunks & {
+  type: "page" | "note";
+  chunks: string[];
+};
+
+export type Metadata = {
+  media?: Array<string>;
+  links?: Array<string>; // idk how ideal this is will figure out after plate js thing
+};
+
+export type SpaceStatus = {
+  type: "inviting" | "invited" | "pending" | "accepted";
+};
+
+export const recommendedQuestionsSchema = z
+  .array(z.string().max(200))
+  .length(10);

apps/backend/src/utils/chunkers.ts

@@ -0,0 +1,116 @@
+import nlp from "compromise";
+
+export default function chunkText(
+  text: string,
+  maxChunkSize: number,
+  overlap: number = 0.2
+): string[] {
+  // Pre-process text to remove excessive whitespace
+  text = text.replace(/\s+/g, " ").trim();
+
+  const sentences = nlp(text).sentences().out("array");
+  const chunks: {
+    text: string;
+    start: number;
+    end: number;
+    metadata?: {
+      position: string;
+      context?: string;
+    };
+  }[] = [];
+
+  let currentChunk: string[] = [];
+  let currentSize = 0;
+
+  for (let i = 0; i < sentences.length; i++) {
+    const sentence = sentences[i].trim();
+
+    // Skip empty sentences
+    if (!sentence) continue;
+
+    // If a single sentence is longer than maxChunkSize, split it
+    if (sentence.length > maxChunkSize) {
+      if (currentChunk.length > 0) {
+        chunks.push({
+          text: currentChunk.join(" "),
+          start: i - currentChunk.length,
+          end: i - 1,
+          metadata: {
+            position: `${i - currentChunk.length}-${i - 1}`,
+            context: currentChunk[0].substring(0, 100), // First 100 chars for context
+          },
+        });
+        currentChunk = [];
+        currentSize = 0;
+      }
+
+      // Split long sentence into smaller chunks
+      const words = sentence.split(" ");
+      let tempChunk: string[] = [];
+
+      for (const word of words) {
+        if (tempChunk.join(" ").length + word.length > maxChunkSize) {
+          chunks.push({
+            text: tempChunk.join(" "),
+            start: i,
+            end: i,
+            metadata: {
+              position: `${i}`,
+              context: "Split sentence",
+            },
+          });
+          tempChunk = [];
+        }
+        tempChunk.push(word);
+      }
+
+      if (tempChunk.length > 0) {
+        chunks.push({
+          text: tempChunk.join(" "),
+          start: i,
+          end: i,
+          metadata: {
+            position: `${i}`,
+            context: "Split sentence remainder",
+          },
+        });
+      }
+      continue;
+    }
+
+    currentChunk.push(sentence);
+    currentSize += sentence.length;
+
+    if (currentSize >= maxChunkSize) {
+      const overlapSize = Math.floor(currentChunk.length * overlap);
+      chunks.push({
+        text: currentChunk.join(" "),
+        start: i - currentChunk.length + 1,
+        end: i,
+        metadata: {
+          position: `${i - currentChunk.length + 1}-${i}`,
+          context: currentChunk[0].substring(0, 100),
+        },
+      });
+
+      // Keep overlap sentences for next chunk
+      currentChunk = currentChunk.slice(-overlapSize);
+      currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0);
+    }
+  }
+
+  // Handle remaining sentences
+  if (currentChunk.length > 0) {
+    chunks.push({
+      text: currentChunk.join(" "),
+      start: sentences.length - currentChunk.length,
+      end: sentences.length - 1,
+      metadata: {
+        position: `${sentences.length - currentChunk.length}-${sentences.length - 1}`,
+        context: currentChunk[0].substring(0, 100),
+      },
+    });
+  }
+
+  return chunks.map((chunk) => chunk.text);
+}

apps/backend/src/utils/cipher.ts

@@ -0,0 +1,79 @@
+async function encrypt(data: string, key: string): Promise<string> {
+  try {
+    const encoder = new TextEncoder();
+    const encodedData = encoder.encode(data);
+
+    const baseForIv = encoder.encode(data + key);
+    const ivHash = await crypto.subtle.digest('SHA-256', baseForIv);
+    const iv = new Uint8Array(ivHash).slice(0, 12);
+
+    const cryptoKey = await crypto.subtle.importKey(
+      "raw",
+      encoder.encode(key),
+      { name: "AES-GCM", length: 256 },
+      false,
+      ["encrypt", "decrypt"]
+    );
+
+    const encrypted = await crypto.subtle.encrypt(
+      { name: "AES-GCM", iv: new Uint8Array(iv).buffer as ArrayBuffer },
+      cryptoKey,
+      encodedData
+    );
+
+    const combined = new Uint8Array([...iv, ...new Uint8Array(encrypted)]);
+
+    // Convert to base64 safely
+    const base64 = Buffer.from(combined).toString("base64");
+
+    // Make URL-safe
+    return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+  } catch (err) {
+    console.error("Encryption error:", err);
+    throw err;
+  }
+}
+
+async function decrypt(encryptedData: string, key: string): Promise<string> {
+  try {
+    // Restore base64 padding and convert URL-safe chars
+    const base64 = encryptedData
+      .replace(/-/g, "+")
+      .replace(/_/g, "/")
+      .padEnd(
+        encryptedData.length + ((4 - (encryptedData.length % 4)) % 4),
+        "="
+      );
+
+    // Use Buffer for safer base64 decoding
+    const combined = Buffer.from(base64, "base64");
+    const combinedArray = new Uint8Array(combined);
+
+    // Extract the IV that was used for encryption
+    const iv = combinedArray.slice(0, 12);
+    const encrypted = combinedArray.slice(12);
+
+    // Import the same key used for encryption
+    const cryptoKey = await crypto.subtle.importKey(
+      "raw",
+      new TextEncoder().encode(key),
+      { name: "AES-GCM", length: 256 },
+      false,
+      ["encrypt", "decrypt"]
+    );
+
+    // Use the extracted IV and key to decrypt
+    const decrypted = await crypto.subtle.decrypt(
+      { name: "AES-GCM", iv: new Uint8Array(iv).buffer as ArrayBuffer },
+      cryptoKey,
+      encrypted.buffer as ArrayBuffer
+    );
+
+    return new TextDecoder().decode(decrypted);
+  } catch (err) {
+    console.error("Decryption error:", err);
+    throw err;
+  }
+}
+
+export { encrypt, decrypt };

apps/backend/src/utils/extractDocumentContent.ts

@@ -0,0 +1,87 @@
+import * as mammoth from "mammoth";
+import { NonRetryableError } from "cloudflare:workflows";
+import { resolvePDFJS } from 'pdfjs-serverless';
+
+interface DocumentContent {
+  content: string;
+  error?: string;
+}
+
+export const extractDocumentContent = async (
+  url: string
+): Promise<DocumentContent> => {
+  try {
+    const fileExtension = url.split(".").pop()?.toLowerCase();
+
+    if (!fileExtension) {
+      throw new Error("Invalid file URL");
+    }
+
+    console.log("file", fileExtension);
+
+    switch (fileExtension) {
+      case "pdf":
+        return await extractPdfContent(url);
+      case "md":
+      case "txt":
+        return await extractTextContent(url);
+      case "doc":
+      case "docx":
+        return await extractWordContent(url);
+      default:
+        throw new NonRetryableError(`Unsupported file type: ${fileExtension}`);
+    }
+  } catch (error) {
+    return {
+      content: "",
+      error: error instanceof Error ? error.message : "Unknown error occurred",
+    };
+  }
+};
+
+async function extractPdfContent(url: string): Promise<DocumentContent> {
+  try {
+    const response = await fetch(url);
+    const arrayBuffer = await response.arrayBuffer();
+
+    // Initialize PDF.js with serverless compatibility
+    const { getDocument } = await resolvePDFJS();
+    
+    // Load the PDF document
+    const pdf = await getDocument({
+      data: arrayBuffer,
+      useSystemFonts: true,
+    }).promise;
+
+    let fullText = "";
+
+    // Extract text from each page
+    for (let i = 1; i <= pdf.numPages; i++) {
+      const page = await pdf.getPage(i);
+      const textContent = await page.getTextContent();
+      const pageText = textContent.items.map((item: any) => item.str).join(" ");
+      fullText += pageText + "\n";
+    }
+
+    return { content: fullText };
+  } catch (error) {
+    console.error("Error extracting PDF content:", error);
+    return {
+      content: "",
+      error: error instanceof Error ? error.message : "Failed to extract PDF content",
+    };
+  }
+}
+
+async function extractTextContent(url: string): Promise<DocumentContent> {
+  const response = await fetch(url);
+  const text = await response.text();
+  return { content: text };
+}
+
+async function extractWordContent(url: string): Promise<DocumentContent> {
+  const response = await fetch(url);
+  const arrayBuffer = await response.arrayBuffer();
+  const result = await mammoth.extractRawText({ arrayBuffer });
+  return { content: result.value };
+}

apps/backend/src/utils/extractor.ts

@@ -0,0 +1,51 @@
+import { Env } from "../types";
+
+export const extractPageContent = async (content: string, env: Env) => {
+  console.log("content", content);
+  const resp = await fetch(`https://md.dhr.wtf?url=${content}`);
+
+  if (!resp.ok) {
+    throw new Error(
+      `Failed to fetch ${content}: ${resp.statusText}` + (await resp.text())
+    );
+  }
+
+  const metadataResp = await fetch(`https://md.dhr.wtf/metadata?url=${content}`);
+
+  if (!metadataResp.ok) {
+    throw new Error(
+      `Failed to fetch metadata for ${content}: ${metadataResp.statusText}` +
+        (await metadataResp.text())
+    );
+  }
+
+  const metadata = await metadataResp.json() as {
+    title?: string;
+    description?: string;
+    image?: string;
+    favicon?: string;
+  };
+
+  const responseText = await resp.text();
+
+  try {
+    const json:  {
+      contentToVectorize: string;
+      contentToSave: string;
+      title?: string;
+      description?: string;
+      image?: string;
+      favicon?: string;
+    } = {
+      contentToSave: responseText,
+      contentToVectorize: responseText,
+      title: metadata.title,
+      description: metadata.description,
+      image: metadata.image,
+      favicon: metadata.favicon,
+    };
+    return json;
+  } catch (e) {
+    throw new Error(`Failed to parse JSON from ${content}: ${e}`);
+  }
+};

apps/backend/src/utils/fetchers.ts

@@ -0,0 +1,146 @@
+import { WorkflowStep } from "cloudflare:workers";
+import { isErr, Ok } from "../errors/results";
+import { typeDecider } from "./typeDecider";
+import { Env, WorkflowParams } from "../types";
+import { unrollTweets } from "./tweetsToThreads";
+import { Tweet } from "react-tweet/api";
+import { NonRetryableError } from "cloudflare:workflows";
+import { extractPageContent } from "./extractor";
+import { extractDocumentContent } from "./extractDocumentContent";
+
+export const fetchContent = async (
+  params: WorkflowParams,
+  env: Env,
+  step: WorkflowStep
+) => {
+  const type = typeDecider(params.content);
+
+  if (isErr(type)) {
+    throw type.error;
+  }
+
+  switch (type.value) {
+    case "page":
+      const pageContent = await step?.do(
+        "extract page content",
+        async () => await extractPageContent(params.content, env)
+      );
+      return {
+        ...pageContent,
+        type: "page",
+      };
+
+    case "tweet":
+      const tweetUrl = new URL(params.content);
+      tweetUrl.search = ""; // Remove all search params
+      const tweetId = tweetUrl.pathname.split("/").pop();
+
+      const unrolledTweetContent = await step.do(
+        "get unrolled tweet content",
+        async () => await unrollTweets(tweetUrl.toString())
+      );
+
+      const rawBaseTweetContent = await step.do(
+        "extract tweet content",
+        async () => {
+          const url = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetId}&lang=en&features=tfw_timeline_list%3A%3Btfw_follower_count_sunset%3Atrue%3Btfw_tweet_edit_backend%3Aon%3Btfw_refsrc_session%3Aon%3Btfw_fosnr_soft_interventions_enabled%3Aon%3Btfw_show_birdwatch_pivots_enabled%3Aon%3Btfw_show_business_verified_badge%3Aon%3Btfw_duplicate_scribes_to_settings%3Aon%3Btfw_use_profile_image_shape_enabled%3Aon%3Btfw_show_blue_verified_badge%3Aon%3Btfw_legacy_timeline_sunset%3Atrue%3Btfw_show_gov_verified_badge%3Aon%3Btfw_show_business_affiliate_badge%3Aon%3Btfw_tweet_edit_frontend%3Aon&token=4c2mmul6mnh`;
+
+          const resp = await fetch(url, {
+            headers: {
+              "User-Agent":
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
+              Accept: "application/json",
+              "Accept-Language": "en-US,en;q=0.5",
+              "Accept-Encoding": "gzip, deflate, br",
+              Connection: "keep-alive",
+              "Upgrade-Insecure-Requests": "1",
+              "Cache-Control": "max-age=0",
+              TE: "Trailers",
+            },
+          });
+
+          const data = (await resp.json()) as Tweet;
+          return data;
+        }
+      );
+
+      let tweetContent: {
+        text: string;
+        metadata: {
+          media?: string[] | undefined;
+          links?: string[] | undefined;
+        };
+        raw: string;
+      };
+
+      if (!unrolledTweetContent || isErr(unrolledTweetContent)) {
+        console.error("Can't get thread, reverting back to single tweet");
+        tweetContent = {
+          text: rawBaseTweetContent.text,
+          metadata: {
+            media: [
+              ...(rawBaseTweetContent.photos?.map((url) => url.expandedUrl) ??
+                []),
+              ...(rawBaseTweetContent.video?.variants[0].src ?? []),
+            ],
+          },
+          raw: `<raw>${JSON.stringify(rawBaseTweetContent)}</raw>`,
+        };
+      } else {
+        tweetContent = {
+          text: unrolledTweetContent.value
+            .map((tweet) => tweet.text)
+            .join("\n"),
+          metadata: {
+            media: unrolledTweetContent.value.flatMap((tweet) => [
+              ...tweet.videos,
+              ...tweet.images,
+            ]),
+            links: unrolledTweetContent.value.flatMap((tweet) => tweet.links),
+          },
+          raw: `<raw>${JSON.stringify(rawBaseTweetContent)}</raw>`,
+        };
+      }
+
+      // make it the same type as the page content
+      const pageContentType: Awaited<ReturnType<typeof extractPageContent>> & {
+        type: string;
+      } = {
+        contentToVectorize:
+          tweetContent.text +
+          "\n\nMetadata for this tweet:\n" +
+          JSON.stringify(tweetContent.metadata) +
+          "\n\nRaw tweet data:\n" +
+          tweetContent.raw,
+        contentToSave: tweetContent.raw,
+        title: "",
+        description: JSON.stringify(tweetContent.metadata),
+        image: "",
+        favicon: "",
+        type: "tweet",
+      };
+      return pageContentType;
+    case "note":
+      const noteContent = {
+        contentToVectorize: params.content,
+        // TODO: different when using platejs
+        contentToSave: params.content,
+        // title is the first 30 characters of the first line
+        title: params.content.split("\n")[0].slice(0, 30),
+        type: "note",
+      };
+      return noteContent;
+    case "document":
+      const documentContent = await step.do(
+        "extract document content",
+        async () => await extractDocumentContent(params.content)
+      );
+      return {
+        contentToVectorize: documentContent.content,
+        contentToSave: documentContent.content,
+        type: "document",
+      };
+    default:
+      throw new NonRetryableError("Unknown content type");
+  }
+};

apps/backend/src/utils/notion.ts

@@ -0,0 +1,239 @@
+interface PageContent {
+  content: string;
+  url: string;
+  title: string;
+  id: string;
+  createdAt: string;
+}
+
+interface NotionBlock {
+  type: string;
+  [key: string]: any;
+}
+
+interface SearchResponse {
+  results: {
+    id: string;
+    object: string;
+    url: string;
+    created_time: string;
+    properties: {
+      title?: {
+        title: Array<{
+          plain_text: string;
+        }>;
+      };
+      Name?: {
+        title: Array<{
+          plain_text: string;
+        }>;
+      };
+    };
+  }[];
+  next_cursor: string | undefined;
+  has_more: boolean;
+}
+
+interface BlockResponse {
+  results: NotionBlock[];
+  next_cursor: string | undefined;
+  has_more: boolean;
+}
+
+export const getAllNotionPageContents = async (
+  token: string,
+  onProgress: (progress: number) => Promise<void>
+): Promise<PageContent[]> => {
+  const pages: PageContent[] = [];
+  const NOTION_API_VERSION = "2022-06-28";
+  const BASE_URL = "https://api.notion.com/v1";
+  const MAX_RETRIES = 3;
+  const BATCH_SIZE = 10; // Number of concurrent requests
+  const PAGE_SIZE = 100; // Number of pages to fetch per search request
+
+  const delay = (ms: number) =>
+    new Promise((resolve) => setTimeout(resolve, ms));
+
+  const notionFetch = async (
+    endpoint: string,
+    options: RequestInit = {},
+    retries = 0
+  ): Promise<any> => {
+    try {
+      const response = await fetch(`${BASE_URL}${endpoint}`, {
+        ...options,
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Notion-Version": NOTION_API_VERSION,
+          "Content-Type": "application/json",
+          ...((options.headers || {}) as Record<string, string>),
+        },
+      });
+
+      if (response.status === 429) {
+        // Rate limit error
+        const retryAfter = parseInt(response.headers.get("Retry-After") || "5");
+        if (retries < MAX_RETRIES) {
+          await delay(retryAfter * 1000);
+          return notionFetch(endpoint, options, retries + 1);
+        }
+      }
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(
+          `Notion API error: ${response.statusText}\n${errorText}`
+        );
+      }
+
+      return response.json();
+    } catch (error) {
+      if (retries < MAX_RETRIES) {
+        await delay(2000 * (retries + 1)); // Exponential backoff
+        return notionFetch(endpoint, options, retries + 1);
+      }
+      throw error;
+    }
+  };
+
+  const convertBlockToMarkdown = (block: NotionBlock): string => {
+    switch (block.type) {
+      case "paragraph":
+        return (
+          block.paragraph?.rich_text
+            ?.map((text: any) => text.plain_text)
+            .join("") || ""
+        );
+      case "heading_1":
+        return `# ${block.heading_1?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      case "heading_2":
+        return `## ${block.heading_2?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      case "heading_3":
+        return `### ${block.heading_3?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      case "bulleted_list_item":
+        return `* ${block.bulleted_list_item?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      case "numbered_list_item":
+        return `1. ${block.numbered_list_item?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      case "to_do":
+        const checked = block.to_do?.checked ? "x" : " ";
+        return `- [${checked}] ${block.to_do?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      case "code":
+        return `\`\`\`${block.code?.language || ""}\n${block.code?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n\`\`\`\n`;
+      case "quote":
+        return `> ${block.quote?.rich_text
+          ?.map((text: any) => text.plain_text)
+          .join("")}\n`;
+      default:
+        return "";
+    }
+  };
+
+  const getAllBlocks = async (pageId: string): Promise<NotionBlock[]> => {
+    const blocks: NotionBlock[] = [];
+    let cursor: string | undefined = undefined;
+
+    do {
+      const endpoint = `/blocks/${pageId}/children${
+        cursor ? `?start_cursor=${cursor}` : ""
+      }`;
+      const response = (await notionFetch(endpoint)) as BlockResponse;
+      blocks.push(...response.results);
+      cursor = response.next_cursor;
+    } while (cursor);
+
+    return blocks;
+  };
+
+  try {
+    let hasMore = true;
+    let cursor: string | undefined = undefined;
+    let allPages: SearchResponse["results"] = [];
+
+    // First, collect all pages
+    while (hasMore) {
+      const searchResponse = (await notionFetch("/search", {
+        method: "POST",
+        body: JSON.stringify({
+          filter: {
+            value: "page",
+            property: "object",
+          },
+          sort: {
+            direction: "ascending",
+            timestamp: "last_edited_time",
+          },
+          start_cursor: cursor,
+          page_size: PAGE_SIZE,
+        }),
+      })) as SearchResponse;
+
+      allPages = [...allPages, ...searchResponse.results];
+      cursor = searchResponse.next_cursor;
+      hasMore = searchResponse.has_more;
+
+      // Report progress for page collection (0-30%)
+      const progressPercent = (allPages.length / (allPages.length + searchResponse.results.length)) * 30;
+      await onProgress(progressPercent);
+    }
+
+    // Process pages in parallel batches
+    for (let i = 0; i < allPages.length; i += BATCH_SIZE) {
+      const batch = allPages.slice(i, i + BATCH_SIZE);
+      const batchResults = await Promise.all(
+        batch.map(async (page) => {
+          try {
+            const blocks = await getAllBlocks(page.id);
+            const pageContent = {
+              content: blocks.map(convertBlockToMarkdown).join("\n"),
+              url: page.url || `https://notion.so/${page.id.replace(/-/g, "")}`,
+              title:
+                page.properties?.Name?.title?.[0]?.plain_text ||
+                page.properties?.title?.title?.[0]?.plain_text ||
+                "Untitled",
+              id: page.id,
+              createdAt: page.created_time,
+            };
+            return pageContent.content.length > 10 ? pageContent : null;
+          } catch (error) {
+            console.error(`Error processing page ${page.id}:`, error);
+            return null;
+          }
+        })
+      );
+
+      pages.push(
+        ...batchResults.filter(
+          (result): result is PageContent => result !== null
+        )
+      );
+
+      // Report progress for page processing (30-100%)
+      const progressPercent = 30 + ((i + BATCH_SIZE) / allPages.length) * 70;
+      await onProgress(Math.min(progressPercent, 100));
+
+      // Add a small delay between batches to respect rate limits
+      if (i + BATCH_SIZE < allPages.length) {
+        await delay(1000);
+      }
+    }
+
+    return pages.filter((page) => page.content.length > 10);
+  } catch (error) {
+    console.error("Error fetching Notion pages:", error);
+    throw error;
+  }
+};

apps/backend/src/utils/tweetsToThreads.ts

@@ -0,0 +1,108 @@
+import * as cheerio from "cheerio";
+import { BaseError } from "../errors/baseError";
+import { Ok, Result } from "../errors/results";
+
+interface Tweet {
+  id: string;
+  text: string;
+  links: Array<string>;
+  images: Array<string>;
+  videos: Array<string>;
+}
+
+class ProcessTweetsError extends BaseError {
+  constructor(message?: string, source?: string) {
+    super("[Thread Proceessing Error]", message, source);
+  }
+}
+
+type TweetProcessResult = Array<Tweet>;
+
+// there won't be a need for url caching right?
+export async function unrollTweets(
+  url: string
+): Promise<Result<TweetProcessResult, ProcessTweetsError>> {
+  const tweetId = url.split("/").pop();
+  const response = await fetch(`https://unrollnow.com/status/${tweetId}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+      "Cache-Control": "max-age=3600",
+    },
+  });
+
+  if (!response.ok) {
+    const error = await response.text();
+    console.error(error);
+    throw new Error(`HTTP error! status: ${response.status} - ${error}`);
+  }
+
+  const html = await response.text();
+  const $ = cheerio.load(html);
+  const tweets: Array<Tweet> = [];
+
+  const urlRegex = /(https?:\/\/\S+)/g;
+  const paragraphs = $(".mainarticle p").toArray();
+
+  const processedTweets = await Promise.all(
+    paragraphs.map(async (element, i) => {
+      const $tweet = $(element);
+      let tweetText = $tweet.text().trim();
+      if (tweetText.length < 1) {
+        return null;
+      }
+
+      if (i === paragraphs.length - 1 && tweetText.toLowerCase() === "yes") {
+        return null;
+      }
+
+      const shortUrls = tweetText.match(urlRegex) || [];
+      console.log("SHORT_URLS_LEN", shortUrls.length);
+      console.log("SHORT_URLS", shortUrls);
+
+      const expandedUrls = await Promise.all(shortUrls.map(expandShortUrl));
+
+      tweetText = tweetText.replace(urlRegex, "").trim().replace(/\s+/g, " ");
+
+      const images = $tweet
+        .nextUntil("p")
+        .find("img.tweetimg")
+        .map((i, img) => $(img).attr("src"))
+        .get();
+
+      const videos = $tweet
+        .nextUntil("p")
+        .find("video > source")
+        .map((i, vid) => $(vid).attr("src"))
+        .get();
+
+      return {
+        id: `${tweetId}_${i}`,
+        text: tweetText,
+        links: expandedUrls,
+        images: images,
+        videos: videos,
+      };
+    })
+  );
+
+  tweets.push(
+    ...processedTweets.filter((tweet): tweet is Tweet => tweet !== null)
+  );
+
+  return Ok(tweets);
+}
+
+async function expandShortUrl(shortUrl: string): Promise<string> {
+  try {
+    const response = await fetch(shortUrl, {
+      method: "HEAD",
+      redirect: "follow",
+    });
+    const expandedUrl = response.url;
+    return expandedUrl;
+  } catch (error) {
+    console.error(`Failed to expand URL: ${shortUrl}`, error);
+    return shortUrl;
+  }
+}

apps/backend/src/utils/typeDecider.ts

@@ -0,0 +1,41 @@
+import { Result, Ok, Err } from "../errors/results";
+import { BaseError } from "../errors/baseError";
+
+export type contentType = "page" | "tweet" | "note" | "document" | "notion";
+
+class GetTypeError extends BaseError {
+  constructor(message?: string, source?: string) {
+    super("[Decide Type Error]", message, source);
+  }
+}
+export const typeDecider = (
+  content: string
+): Result<contentType, GetTypeError> => {
+  try {
+    // if the content is a URL, then it's a page. if its a URL with https://x.com/user/status/123, then it's a tweet.
+    // if it ends with .pdf etc then it's a document. else, it's a note.
+    // do strict checking with regex
+    if (
+      content.match(/https?:\/\/(x\.com|twitter\.com)\/[\w]+\/[\w]+\/[\d]+/)
+    ) {
+      return Ok("tweet");
+    } else if (content.match(/\.(pdf|doc|docx|txt|rtf|odt|md)/i)) {
+      return Ok("document"); 
+    } else if (
+      content.match(/https?:\/\/(www\.)?notion\.so\/.*/)
+    ) {
+      return Ok("notion");
+    } else if (
+      content.match(
+        /^(https?:\/\/)?(www\.)?[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,5}(\/.*)?$/i
+      )
+    ) {
+      return Ok("page");
+    } else {
+      return Ok("note");
+    }
+  } catch (e) {
+    console.error("[Decide Type Error]", e);
+    return Err(new GetTypeError((e as Error).message, "typeDecider"));
+  }
+};

apps/backend/src/workflow/index.ts

@@ -0,0 +1,237 @@
+import {
+  WorkflowEntrypoint,
+  WorkflowStep,
+  WorkflowEvent,
+} from "cloudflare:workers";
+import { Env, WorkflowParams } from "../types";
+import { fetchContent } from "../utils/fetchers";
+import chunkText from "../utils/chunkers";
+import { database, eq, inArray } from "@supermemory/db";
+import {
+  ChunkInsert,
+  contentToSpace,
+  documents,
+  spaces,
+} from "@supermemory/db/schema";
+import { embedMany } from "ai";
+import { openai } from "../providers";
+import { chunk } from "@supermemory/db/schema";
+import { NonRetryableError } from "cloudflare:workflows";
+
+// TODO: handle errors properly here.
+
+export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
+  async run(event: WorkflowEvent<WorkflowParams>, step: WorkflowStep) {
+    // Step 0: Check if user has reached memory limit
+    await step.do("check memory limit", async () => {
+      const existingMemories = await database(this.env.HYPERDRIVE.connectionString)
+        .select()
+        .from(documents)
+        .where(eq(documents.userId, event.payload.userId));
+
+      if (existingMemories.length >= 2000) {
+        await database(this.env.HYPERDRIVE.connectionString)
+          .delete(documents)
+          .where(eq(documents.uuid, event.payload.uuid));
+        throw new NonRetryableError("You have reached the maximum limit of 2000 memories");
+      }
+    });
+
+    // Step 1: Get and format the content.
+    const rawContent =
+      event.payload.prefetched ??
+      (await step.do(
+        "fetch content",
+        async () => await fetchContent(event.payload, this.env, step)
+      ));
+
+    // check that the rawcontent is not too big
+    if (rawContent.contentToVectorize.length > 100000) {
+      await database(this.env.HYPERDRIVE.connectionString)
+        .delete(documents)
+        .where(eq(documents.uuid, event.payload.uuid));
+      throw new NonRetryableError("The content is too big (maximum 20 pages)");
+    }
+
+    const chunked = await step.do("chunk content", async () =>
+      chunkText(rawContent.contentToVectorize, 768)
+    );
+
+    // Step 2: Create the document in the database.
+    const document = await step.do("create document", async () => {
+      try {
+        // First check if document exists
+        const existingDoc = await database(this.env.HYPERDRIVE.connectionString)
+          .select()
+          .from(documents)
+          .where(eq(documents.uuid, event.payload.uuid))
+          .limit(1);
+
+        return await database(this.env.HYPERDRIVE.connectionString)
+          .insert(documents)
+          .values({
+            userId: event.payload.userId,
+            type: event.payload.type,
+            uuid: event.payload.uuid,
+            ...(event.payload.url && { url: event.payload.url }),
+            title: rawContent.title,
+            content: rawContent.contentToSave,
+            description:
+              "description" in rawContent
+                ? (rawContent.description ?? "")
+                : (event.payload.prefetched?.description ?? undefined),
+            ogImage:
+              "image" in rawContent
+                ? (rawContent.image ?? "")
+                : (event.payload.prefetched?.ogImage ?? undefined),
+            raw: rawContent.contentToVectorize,
+            isSuccessfullyProcessed: false,
+            updatedAt: new Date(),
+            ...(event.payload.createdAt && {
+              createdAt: new Date(event.payload.createdAt),
+            }),
+          })
+          .onConflictDoUpdate({
+            target: documents.uuid,
+            set: {
+              title: rawContent.title,
+              content: rawContent.contentToSave,
+              description:
+                "description" in rawContent
+                  ? (rawContent.description ?? "")
+                  : (event.payload.prefetched?.description ?? undefined),
+              ogImage:
+                "image" in rawContent
+                  ? (rawContent.image ?? "")
+                  : (event.payload.prefetched?.ogImage ?? undefined),
+              raw: rawContent.contentToVectorize,
+              isSuccessfullyProcessed: false,
+              updatedAt: new Date(),
+            },
+          })
+          .returning();
+      } catch (error) {
+        console.log("here's the error", error);
+        // Check if error is a unique constraint violation
+        if (
+          error instanceof Error &&
+          error.message.includes("document_url_user_id_idx")
+        ) {
+          // Document already exists for this user, stop workflow
+          await database(this.env.HYPERDRIVE.connectionString)
+            .delete(documents)
+            .where(eq(documents.uuid, event.payload.uuid));
+          throw new NonRetryableError("Document already exists for this user");
+        }
+        if (
+          error instanceof Error &&
+          error.message.includes("document_raw_user_idx")
+        ) {
+          await database(this.env.HYPERDRIVE.connectionString)
+            .delete(documents)
+            .where(eq(documents.uuid, event.payload.uuid));
+          throw new NonRetryableError("The exact same document already exists");
+        }
+        throw error; // Re-throw other errors
+      }
+    });
+
+    if (!document || document.length === 0) {
+      throw new Error(
+        "Failed to create/update document - no document returned"
+      );
+    }
+
+    const model = openai(this.env, this.env.OPEN_AI_API_KEY).embedding(
+      "text-embedding-3-large",
+      {
+        dimensions: 1536,
+      }
+    );
+
+    // Step 3: Create chunks from the content.
+    const embeddings = await step.do(
+      "create embeddings",
+      {
+        retries: {
+          backoff: "constant",
+          delay: "10 seconds",
+          limit: 7,
+        },
+        timeout: "2 minutes",
+      },
+      async () => {
+        const { embeddings }: { embeddings: Array<number>[] } = await embedMany(
+          {
+            model,
+            values: chunked,
+          }
+        );
+
+        return embeddings;
+      }
+    );
+
+    // Step 4: Prepare chunk data
+    const chunkInsertData: ChunkInsert[] = await step.do(
+      "prepare chunk data",
+      async () =>
+        chunked.map((chunk, index) => ({
+          documentId: document[0].id,
+          textContent: chunk,
+          orderInDocument: index,
+          embeddings: embeddings[index],
+        }))
+    );
+
+    console.log(chunkInsertData);
+
+    // Step 5: Insert chunks
+    if (chunkInsertData.length > 0) {
+      await step.do("insert chunks", async () =>
+        database(this.env.HYPERDRIVE.connectionString).transaction(
+          async (trx) => {
+            await trx.insert(chunk).values(chunkInsertData);
+          }
+        )
+      );
+    }
+
+    // step 6: add content to spaces
+    if (event.payload.spaces) {
+      await step.do("add content to spaces", async () => {
+        await database(this.env.HYPERDRIVE.connectionString).transaction(
+          async (trx) => {
+            // First get the space IDs from the UUIDs
+            const spaceIds = await trx
+              .select({ id: spaces.id })
+              .from(spaces)
+              .where(inArray(spaces.uuid, event.payload.spaces ?? []));
+
+            if (spaceIds.length === 0) {
+              return;
+            }
+
+            // Then insert the content-space mappings using the actual space IDs
+            await trx.insert(contentToSpace).values(
+              spaceIds.map((space) => ({
+                contentId: document[0].id,
+                spaceId: space.id,
+              }))
+            );
+          }
+        );
+      });
+    }
+
+    // Step 7: Mark the document as successfully processed
+    await step.do("mark document as successfully processed", async () => {
+      await database(this.env.HYPERDRIVE.connectionString)
+        .update(documents)
+        .set({
+          isSuccessfullyProcessed: true,
+        })
+        .where(eq(documents.id, document[0].id));
+    });
+  }
+}

apps/backend/tailwind.config.js

@@ -0,0 +1,9 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: ["./src/**/*.{js,ts,jsx,tsx}"],
+  theme: {
+    extend: {},
+  },
+  plugins: [],
+}
+

apps/backend/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "skipLibCheck": true,
+    "lib": ["ESNext"],
+    "types": [
+      "@cloudflare/workers-types/experimental",
+      "@cloudflare/workers-types"
+    ],
+    "jsx": "react-jsx",
+    "jsxImportSource": "hono/jsx"
+  }
+}

apps/backend/types.d.ts

@@ -0,0 +1,3 @@
+declare module "@mixmark-io/domino" {
+    export function createDocument(html: string): Document;
+  }

apps/backend/wrangler.toml

@@ -0,0 +1,52 @@
+name = "supermemory-backend"
+main = "src/index.tsx"
+compatibility_date = "2024-10-11"
+compatibility_flags = [ "nodejs_compat" ]
+
+[assets]
+directory = "./public/"
+binding = "ASSETS"
+
+
+[observability]
+enabled = true
+
+[placement]
+mode = "smart"
+
+[[workflows]]
+name = "content-workflow-supermemory"
+binding = "CONTENT_WORKFLOW"
+class_name = "ContentWorkflow"
+
+
+[[kv_namespaces]]
+binding= "MD_CACHE"
+id = "3186489f943d409a9b772d876a58a73e"
+preview_id = "3186489f943d409a9b772d876a58a73e"
+
+[[kv_namespaces]]
+binding = "ENCRYPTED_TOKENS"
+id = "a1f048ee14644468ad63b817b5648a31"
+preview_id = "a1f048ee14644468ad63b817b5648a31"
+
+[[hyperdrive]]
+binding = "HYPERDRIVE"
+id = "3a377d1b9c084e698ee201f10dfa8131"
+localConnectionString = "postgres://postgres:postgres@localhost:5432/supermemorydhravya?sslmode=require"
+
+[[unsafe.bindings]]
+name = "EMAIL_LIMITER"
+type = "ratelimit"
+namespace_id = "2114284"
+simple = { limit = 1, period = 60 }
+
+tail_consumers = [{service = "supermemory-backend-tail"}]
+
+[[durable_objects.bindings]]
+name = "RATE_LIMITER"
+class_name = "DurableObjectRateLimiter"
+
+[[migrations]]
+tag = "v1"
+new_classes = ["DurableObjectRateLimiter"]

apps/browser-rendering

@@ -1 +0,0 @@
-Subproject commit f8878349d74dc17dfb938479a68948a12c39b724

apps/cf-ai-backend/.gitignore

@@ -1,13 +0,0 @@
-dist
-node_modules
-worker
-package-lock.json
-yarn.lock
-.cargo-ok
-
-# Output of 'npm pack'
-*.tgz
-
-# wrangler files
-.wrangler
-.dev.vars

apps/cf-ai-backend/README.md

@@ -1,50 +0,0 @@
-baseURL: https://new-cf-ai-backend.dhravya.workers.dev
-
-Authentication:
-You must authenticate with a header and `Authorization: bearer token` for each request in `/api/*` routes.
-
-### Add content:
-
-POST `/api/add` with
-
-```
-body {
-  pageContent: z.string(),
-  title: z.string().optional(),
-  description: z.string().optional(),
-  space: z.string().optional(),
-  url: z.string(),
-  user: z.string(),
-}
-```
-
-### Query without user data
-
-GET `/api/ask` with
-query `?query=testing`
-
-(this is temp but works perfectly, will change soon for chat use cases specifically)
-
-### Query vectorize and get results in natural language
-
-POST `/api/chat` with
-
-```
-query paramters (?query=...&" {
-      query: z.string(),
-      topK: z.number().optional().default(10),
-      user: z.string(),
-      spaces: z.string().optional(),
-      sourcesOnly: z.string().optional().default("false"),
-      model: z.string().optional().default("gpt-4o"),
-    }
-
-body z.object({
-  chatHistory: z.array(contentObj).optional(),
-});
-```
-
-### Delete vectors
-
-DELETE `/api/delete` with
-query param websiteUrl, user

apps/cf-ai-backend/package.json

@@ -1,23 +0,0 @@
-{
-	"name": "new-cf-ai-backend",
-	"private": true,
-	"version": "0.0.1",
-	"main": "src/index.ts",
-	"scripts": {
-		"test": "jest --verbose",
-		"deploy": "wrangler deploy",
-		"dev": "wrangler dev --remote --port 8686",
-		"start": "wrangler dev",
-		"unsafe-reset-vector-db": "wrangler vectorize delete  supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine"
-	},
-	"license": "MIT",
-	"dependencies": {
-		"@hono/zod-validator": "^0.2.1",
-		"hono": "^4.5.1",
-		"honox": "^0.1.23",
-		"vite": "^5.3.5"
-	},
-	"devDependencies": {
-		"@cloudflare/workers-types": "^4.20240614.0"
-	}
-}

apps/cf-ai-backend/src/db/index.ts

@@ -1,7 +0,0 @@
-import { drizzle } from "drizzle-orm/d1";
-import { Env } from "../types";
-
-import * as schema from "@repo/db/schema";
-
-export const database = (env: Env) =>
-	drizzle(env.DATABASE, { schema, logger: true });

apps/cf-ai-backend/src/helper.ts

@@ -1,385 +0,0 @@
-import { Context } from "hono";
-import { Env, vectorObj, Chunks } from "./types";
-import { CloudflareVectorizeStore } from "@langchain/cloudflare";
-import { OpenAIEmbeddings } from "./utils/OpenAIEmbedder";
-import { createOpenAI } from "@ai-sdk/openai";
-import { createGoogleGenerativeAI } from "@ai-sdk/google";
-import { createAnthropic } from "@ai-sdk/anthropic";
-import { z } from "zod";
-import { seededRandom } from "./utils/seededRandom";
-import { bulkInsertKv } from "./utils/kvBulkInsert";
-
-export async function initQuery(env: Env, model: string = "gemini-1.5-pro") {
-	const embeddings = new OpenAIEmbeddings({
-		apiKey: env.OPENAI_API_KEY,
-		modelName: "text-embedding-3-small",
-	});
-
-	const store = new CloudflareVectorizeStore(embeddings, {
-		index: env.VECTORIZE_INDEX,
-	});
-
-	let selectedModel:
-		| ReturnType<ReturnType<typeof createOpenAI>>
-		| ReturnType<ReturnType<typeof createGoogleGenerativeAI>>
-		| ReturnType<ReturnType<typeof createAnthropic>>;
-
-	switch (model) {
-		case "claude-3-opus":
-			const anthropic = createAnthropic({
-				apiKey: env.ANTHROPIC_API_KEY,
-				baseURL:
-					"https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/anthropic",
-			});
-			selectedModel = anthropic.chat("claude-3-opus-20240229");
-			console.log("Selected model: ", selectedModel);
-			break;
-		case "gemini-1.5-pro":
-			console.log("YES GOOGLE");
-			const googleai = createGoogleGenerativeAI({
-				apiKey: env.GOOGLE_AI_API_KEY,
-			});
-			selectedModel = googleai.chat("models/gemini-1.5-flash-8b");
-			console.log("Selected model: ", selectedModel);
-			break;
-		case "gpt-4o":
-		default:
-			const openai = createOpenAI({
-				apiKey: env.OPENAI_API_KEY,
-				baseURL:
-					"https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai",
-			});
-			selectedModel = openai.chat("gpt-4o-mini");
-			break;
-	}
-
-	return { store, model: selectedModel };
-}
-
-export async function deleteDocument({
-	url,
-	user,
-	c,
-	store,
-}: {
-	url: string;
-	user: string;
-	c: Context<{ Bindings: Env }>;
-	store: CloudflareVectorizeStore;
-}) {
-	const toBeDeleted = `${url}#supermemory-web`;
-	const random = seededRandom(toBeDeleted);
-
-	const uuid =
-		random().toString(36).substring(2, 15) +
-		random().toString(36).substring(2, 15);
-
-	const allIds = await c.env.KV.list({ prefix: uuid });
-
-	if (allIds.keys.length > 0) {
-		const savedVectorIds = allIds.keys.map((key) => key.name);
-		const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds);
-		// We don't actually delete document directly, we just remove the user from the metadata.
-		// If there's no user left, we can delete the document.
-		const newVectors = vectors.map((vector) => {
-			delete vector.metadata[`user-${user}`];
-
-			// Get count of how many users are left
-			const userCount = Object.keys(vector.metadata).filter((key) =>
-				key.startsWith("user-"),
-			).length;
-
-			// If there's no user left, we can delete the document.
-			// need to make sure that every chunk is deleted otherwise it would be problematic.
-			if (userCount === 0) {
-				store.delete({ ids: savedVectorIds });
-				void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id)));
-				return null;
-			}
-
-			return vector;
-		});
-
-		// If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors.
-		if (newVectors.every((v) => v === null)) {
-			await c.env.KV.delete(uuid);
-		} else {
-			await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null));
-		}
-	}
-}
-
-function sanitizeKey(key: string): string {
-	if (!key) throw new Error("Key cannot be empty");
-
-	// Remove or replace invalid characters
-	let sanitizedKey = key.replace(/[.$"]/g, "_");
-
-	// Ensure key does not start with $
-	if (sanitizedKey.startsWith("$")) {
-		sanitizedKey = sanitizedKey.substring(1);
-	}
-
-	return sanitizedKey;
-}
-
-export async function batchCreateChunksAndEmbeddings({
-	store,
-	body,
-	chunks,
-	env: env,
-}: {
-	store: CloudflareVectorizeStore;
-	body: z.infer<typeof vectorObj>;
-	chunks: Chunks;
-	env: Env;
-}) {
-	//! NOTE that we use #supermemory-web to ensure that
-	//! If a user saves it through the extension, we don't want other users to be able to see it.
-	// Requests from the extension should ALWAYS have a unique ID with the USERiD in it.
-	// I cannot stress this enough, important for security.
-
-	const ourID = `${body.url}#supermemory-web`;
-	const random = seededRandom(ourID);
-	const uuid =
-		random().toString(36).substring(2, 15) +
-		random().toString(36).substring(2, 15);
-
-	const allIds = await env.KV.list({ prefix: uuid });
-
-	// If some chunks for that content already exist, we'll just update the metadata to include
-	// the user.
-	if (allIds.keys.length > 0) {
-		const savedVectorIds = allIds.keys.map((key) => key.name);
-		const vectors = [];
-		//Search in a batch of 20
-		for (let i = 0; i < savedVectorIds.length; i += 20) {
-			const batch = savedVectorIds.slice(i, i + 20);
-			const batchVectors = await env.VECTORIZE_INDEX.getByIds(batch);
-			vectors.push(...batchVectors);
-		}
-		console.log(
-			"vector Id list: ",
-			vectors.map((vector) => {
-				return vector.id;
-			}),
-		);
-		// Now, we'll update all vector metadatas with one more userId and all spaceIds
-		const newVectors = vectors.map((vector) => {
-			console.log(JSON.stringify(vector.metadata));
-			vector.metadata = {
-				...vector.metadata,
-				[`user-${body.user}`]: 1,
-
-				// For each space in body, add the spaceId to the vector metadata
-				...(body.spaces ?? [])?.reduce((acc, space) => {
-					acc[`space-${body.user}-${space}`] = 1;
-					return acc;
-				}, {}),
-			};
-			return vector;
-		});
-
-		// upsert in batch of 20
-		const results = [];
-		for (let i = 0; i < newVectors.length; i += 20) {
-			results.push(newVectors.slice(i, i + 20));
-			console.log(newVectors);
-		}
-
-		await Promise.all(
-			results.map((result) => {
-				return env.VECTORIZE_INDEX.upsert(result);
-			}),
-		);
-		return;
-	}
-
-	switch (chunks.type) {
-		case "tweet":
-			{
-				const commonMetaData = {
-					type: body.type ?? "tweet",
-					title: body.title?.slice(0, 50) ?? "",
-					description: body.description?.slice(0, 50) ?? "",
-					url: body.url,
-					[sanitizeKey(`user-${body.user}`)]: 1,
-				};
-
-				const spaceMetadata = body.spaces?.reduce((acc, space) => {
-					acc[`space-${body.user}-${space}`] = 1;
-					return acc;
-				}, {});
-
-				const ids = [];
-				const preparedDocuments = chunks.chunks
-					.map((tweet, i) => {
-						return tweet.chunkedTweet.map((chunk) => {
-							const id = `${uuid}-${i}`;
-							ids.push(id);
-							const { tweetLinks, tweetVids, tweetId, tweetImages } =
-								tweet.metadata;
-							return {
-								pageContent: chunk,
-								metadata: {
-									content: chunk,
-									links: tweetLinks,
-									videos: tweetVids,
-									tweetId: tweetId,
-									tweetImages: tweetImages,
-									...commonMetaData,
-									...spaceMetadata,
-								},
-							};
-						});
-					})
-					.flat();
-
-				const docs = await store.addDocuments(preparedDocuments, {
-					ids: ids,
-				});
-				console.log("these are the doucment ids", ids);
-				console.log("Docs added:", docs);
-				const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env;
-				await bulkInsertKv(
-					{ CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID },
-					{ chunkIds: ids, urlid: ourID },
-				);
-			}
-			break;
-		case "page":
-			{
-				const commonMetaData = {
-					type: body.type ?? "page",
-					title: body.title?.slice(0, 50) ?? "",
-					description: body.description?.slice(0, 50) ?? "",
-					url: body.url,
-					[sanitizeKey(`user-${body.user}`)]: 1,
-				};
-				const spaceMetadata = body.spaces?.reduce((acc, space) => {
-					acc[`space-${body.user}-${space}`] = 1;
-					return acc;
-				}, {});
-
-				const ids = [];
-				console.log("Page hit moving on to the for loop");
-				for (let i = 0; i < chunks.chunks.length; i++) {
-					const chunk = chunks.chunks[i];
-					const id = `${uuid}-${i}`;
-					ids.push(id);
-					const document = {
-						pageContent: chunk,
-						metadata: {
-							...commonMetaData,
-							...spaceMetadata,
-						},
-					};
-					const docs = await store.addDocuments([document], { ids: [id] });
-					console.log("Docs added:", docs);
-					// Wait for a second after every 20 documents for open ai rate limit
-					console.log(
-						"This is the 20th thing in the list?",
-						(i + 1) % 20 === 0,
-					);
-					if ((i + 1) % 20 === 0) {
-						await new Promise((resolve) => setTimeout(resolve, 1000));
-					}
-				}
-
-				const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env;
-				await bulkInsertKv(
-					{ CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID },
-					{ chunkIds: ids, urlid: ourID },
-				);
-			}
-			break;
-		case "note":
-			{
-				const commonMetaData = {
-					title: body.title?.slice(0, 50) ?? "",
-					type: body.type ?? "page",
-					description: body.description?.slice(0, 50) ?? "",
-					url: body.url,
-					[sanitizeKey(`user-${body.user}`)]: 1,
-				};
-				const spaceMetadata = body.spaces?.reduce((acc, space) => {
-					acc[`space-${body.user}-${space}`] = 1;
-					return acc;
-				}, {});
-
-				const ids = [];
-				for (let i = 0; i < chunks.chunks.length; i++) {
-					const chunk = chunks.chunks[i];
-					const id = `${uuid}-${i}`;
-					ids.push(id);
-					const document = {
-						pageContent: chunk,
-						metadata: {
-							...commonMetaData,
-							...spaceMetadata,
-						},
-					};
-					const docs = await store.addDocuments([document], { ids: [id] });
-					console.log("Docs added:", docs);
-					// Wait for a second after every 20 documents for open ai rate limit
-					console.log(
-						"This is the 20th thing in the list?",
-						(i + 1) % 20 === 0,
-					);
-					if ((i + 1) % 20 === 0) {
-						await new Promise((resolve) => setTimeout(resolve, 1000));
-					}
-				}
-
-				const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env;
-				await bulkInsertKv(
-					{ CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID },
-					{ chunkIds: ids, urlid: ourID },
-				);
-			}
-			break;
-		case "image": {
-			const commonMetaData = {
-				type: body.type ?? "image",
-				title: body.title,
-				description: body.description?.slice(0, 50) ?? "",
-				url: body.url,
-				[sanitizeKey(`user-${body.user}`)]: 1,
-			};
-			const spaceMetadata = body.spaces?.reduce((acc, space) => {
-				acc[`space-${body.user}-${space}`] = 1;
-				return acc;
-			}, {});
-
-			const ids = [];
-			for (let i = 0; i < chunks.chunks.length; i++) {
-				const chunk = chunks.chunks[i];
-				const id = `${uuid}-${i}`;
-				ids.push(id);
-				const document = {
-					pageContent: chunk,
-					metadata: {
-						...commonMetaData,
-						...spaceMetadata,
-					},
-				};
-				const docs = await store.addDocuments([document], { ids: [id] });
-				console.log("Docs added:", docs);
-				// Wait for a second after every 20 documents for open ai rate limit
-				console.log("This is the 20th thing in the list?", (i + 1) % 20 === 0);
-				if ((i + 1) % 20 === 0) {
-					console.log("-----------waiting atm");
-					await new Promise((resolve) => setTimeout(resolve, 1000));
-				}
-			}
-
-			const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env;
-			await bulkInsertKv(
-				{ CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID },
-				{ chunkIds: ids, urlid: ourID },
-			);
-		}
-	}
-
-	return;
-}

apps/cf-ai-backend/src/index.ts

@@ -1,781 +0,0 @@
-import { boolean, z } from "zod";
-import { Hono } from "hono";
-import { CoreMessage, generateText, streamText, tool } from "ai";
-import {
-	chatObj,
-	Chunks,
-	Env,
-	ImageChunks,
-	PageOrNoteChunks,
-	TweetChunks,
-	vectorObj,
-	vectorBody,
-} from "./types";
-import {
-	batchCreateChunksAndEmbeddings,
-	deleteDocument,
-	initQuery,
-} from "./helper";
-import { timing } from "hono/timing";
-import { logger } from "hono/logger";
-import { poweredBy } from "hono/powered-by";
-import { bearerAuth } from "hono/bearer-auth";
-import { zValidator } from "@hono/zod-validator";
-import chunkText from "./queueConsumer/chunkers/chonker";
-import { systemPrompt, template } from "./prompts/prompt1";
-import { swaggerUI } from "@hono/swagger-ui";
-import { database } from "./db";
-import { storedContent } from "@repo/db/schema";
-import { sql, and, eq } from "drizzle-orm";
-import { LIMITS } from "@repo/shared-types";
-import { typeDecider } from "./queueConsumer/utils/typeDecider";
-// import { chunkThread } from "./utils/chunkTweet";
-import {
-	chunkNote,
-	chunkPage,
-} from "./queueConsumer/chunkers/chunkPageOrNotes";
-import { queue } from "./queueConsumer";
-import { isErr } from "./errors/results";
-import { createOpenAI } from "@ai-sdk/openai";
-
-const app = new Hono<{ Bindings: Env }>();
-
-// ------- MIDDLEWARES -------
-app.use("*", poweredBy());
-app.use("*", timing());
-app.use("*", logger());
-
-app.use("/api/", async (c, next) => {
-	if (c.env.NODE_ENV !== "development") {
-		const auth = bearerAuth({ token: c.env.SECURITY_KEY });
-		return auth(c, next);
-	}
-	return next();
-});
-// ------- MIDDLEWARES END -------
-
-const fileSchema = z
-	.instanceof(File)
-	.refine(
-		(file) => file.size <= 10 * 1024 * 1024,
-		"File size should be less than 10MB",
-	) // Validate file size
-	.refine(
-		(file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type),
-		"Invalid file type",
-	); // Validate file type
-
-app.get("/", (c) => {
-	return c.text("Supermemory backend API is running!");
-});
-
-app.get("/api/health", (c) => {
-	return c.json({ status: "ok" });
-});
-
-app.post("/api/add", zValidator("json", vectorBody), async (c) => {
-	try {
-		const body = c.req.valid("json");
-		//This is something I don't like
-		// console.log("api/add hit!!!!");
-		//Have to do limit on this also duplicate check here
-		const db = database(c.env);
-		const typeResult = typeDecider(body.url);
-
-		const saveToDbUrl =
-			(body.url.split("#supermemory-user-")[0] ?? body.url) + // Why does this have to be a split from #supermemory-user?
-			"#supermemory-user-" +
-			body.user;
-
-		const alreadyExist = await db
-			.select()
-			.from(storedContent)
-			.where(eq(storedContent.baseUrl, saveToDbUrl));
-
-		if (alreadyExist.length > 0) {
-			return c.json({ status: "error", message: "the content already exists" });
-		}
-
-		if (isErr(typeResult)) {
-			throw typeResult.error;
-		}
-		// limiting in the backend
-		const type = typeResult.value;
-		const countResult = await db
-			.select({
-				count: sql<number>`count(*)`.mapWith(Number),
-			})
-			.from(storedContent)
-			.where(
-				and(eq(storedContent.userId, body.user), eq(storedContent.type, type)),
-			);
-
-		const currentCount = countResult[0]?.count || 0;
-		const totalLimit = LIMITS[type as keyof typeof LIMITS];
-		const remainingLimit = totalLimit - currentCount;
-		const items = 1;
-		const isWithinLimit = items <= remainingLimit;
-
-		// unique contraint check
-
-		if (isWithinLimit) {
-			await c.env.EMBEDCHUNKS_QUEUE.send({
-				content: saveToDbUrl,
-				user: body.user,
-				space: body.spaces,
-				type: type,
-			});
-		} else {
-			return c.json({
-				status: "error",
-				message:
-					"You have exceed the current limit for this type of document, please try removing something form memories ",
-			});
-		}
-
-		return c.json({ status: "ok" });
-	} catch (error) {
-		console.error("Error processing request:", error);
-		return c.json({ status: "error", message: error.message }, 500);
-	}
-});
-
-app.post(
-	"/api/add-with-image",
-	zValidator(
-		"form",
-		z.object({
-			images: z
-				.array(fileSchema)
-				.min(1, "At least one image is required")
-				.optional(),
-			"images[]": z
-				.array(fileSchema)
-				.min(1, "At least one image is required")
-				.optional(),
-			text: z.string().optional(),
-			spaces: z.array(z.string()).optional(),
-			url: z.string(),
-			user: z.string(),
-		}),
-		(c) => {
-			console.log(c);
-		},
-	),
-	async (c) => {
-		const body = c.req.valid("form");
-
-		const { store } = await initQuery(c.env);
-
-		if (!(body.images || body["images[]"])) {
-			return c.json({ status: "error", message: "No images found" }, 400);
-		}
-
-		const imagePromises = (body.images ?? body["images[]"]).map(
-			async (image) => {
-				const buffer = await image.arrayBuffer();
-				const input = {
-					image: [...new Uint8Array(buffer)],
-					prompt:
-						"What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.",
-					max_tokens: 1024,
-				};
-				const response = await c.env.AI.run(
-					"@cf/llava-hf/llava-1.5-7b-hf",
-					input,
-				);
-				console.log(response.description);
-				return response.description;
-			},
-		);
-
-		const imageDescriptions = await Promise.all(imagePromises);
-		const chunks: ImageChunks = {
-			type: "image",
-			chunks: [
-				imageDescriptions,
-				...(body.text ? chunkText(body.text, 1536) : []),
-			].flat(),
-		};
-
-		await batchCreateChunksAndEmbeddings({
-			store,
-			body: {
-				url: body.url,
-				user: body.user,
-				type: "image",
-				description:
-					imageDescriptions.length > 1
-						? `A group of ${imageDescriptions.length} images on ${body.url}`
-						: imageDescriptions[0],
-				spaces: body.spaces,
-				pageContent: imageDescriptions.join("\n"),
-				title: "Image content from the web",
-			},
-			chunks: chunks,
-			env: c.env,
-		});
-
-		return c.json({ status: "ok" });
-	},
-);
-
-app.get(
-	"/api/ask",
-	zValidator(
-		"query",
-		z.object({
-			query: z.string(),
-		}),
-	),
-	async (c) => {
-		const query = c.req.valid("query");
-
-		const { model } = await initQuery(c.env);
-
-		const response = await streamText({ model, prompt: query.query });
-		const r = response.toTextStreamResponse();
-
-		return r;
-	},
-);
-
-app.get(
-	"/api/search",
-	zValidator("query", z.object({ query: z.string(), user: z.string() })),
-	async (c) => {
-		const { query, user } = c.req.valid("query");
-		const filter: VectorizeVectorMetadataFilter = {
-			[`user-${user}`]: 1,
-		};
-
-		const { store } = await initQuery(c.env);
-		const queryAsVector = await store.embeddings.embedQuery(query);
-
-		const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, {
-			topK: 5,
-			filter,
-			returnMetadata: true,
-		});
-
-		const minScore = Math.min(...resp.matches.map(({ score }) => score));
-		const maxScore = Math.max(...resp.matches.map(({ score }) => score));
-
-		// This entire chat part is basically just a dumb down version of the /api/chat endpoint.
-		const normalizedData = resp.matches.map((data) => ({
-			...data,
-			normalizedScore:
-				maxScore !== minScore
-					? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98
-					: 50,
-		}));
-
-		const preparedContext = normalizedData.map(
-			({ metadata, score, normalizedScore }) => ({
-				context: `Title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`,
-				score,
-				normalizedScore,
-			}),
-		);
-
-		return c.json({
-			status: "ok",
-			response: preparedContext,
-		});
-	},
-);
-
-// This is a special endpoint for our "chatbot-only" solutions.
-// It does both - adding content AND chatting with it.
-app.post(
-	"/api/autoChatOrAdd",
-	zValidator(
-		"query",
-		z.object({
-			query: z.string(),
-			user: z.string(),
-		}),
-	),
-	zValidator("json", chatObj),
-	async (c) => {
-		const { query, user } = c.req.valid("query");
-		const { chatHistory } = c.req.valid("json");
-
-		const openai = createOpenAI({
-			apiKey: c.env.OPENAI_API_KEY,
-		});
-		const model = openai.chat("gpt-4o-mini");
-		const { store } = await initQuery(c.env);
-
-		let task: "add" | "chat" = "chat";
-		let thingToAdd: "page" | "image" | "text" | undefined = undefined;
-		let addContent: string | undefined = undefined;
-
-		// This is a "router". this finds out if the user wants to add a document, or chat with the AI to get a response.
-		const routerQuery = await generateText({
-			model: model,
-			system: `You are Supermemory chatbot. You can either add a document to the supermemory database, or return a chat response. Based on this query,
-        You must determine what to do. Basically if it feels like a "question", then you should intiate a chat. If it feels like a "command" or feels like something that could be forwarded to the AI, then you should add a document.
-        You must also extract the "thing" to add and what type of thing it is.`,
-			prompt: `Question from user: ${query}`,
-			tools: {
-				decideTask: tool({
-					description:
-						"Decide if the user wants to add a document or chat with the AI",
-					parameters: z.object({
-						generatedTask: z.enum(["add", "chat"]),
-						contentToAdd: z
-							.object({
-								thing: z.enum(["page", "image", "text"]),
-								content: z.string(),
-							})
-							.optional(),
-					}),
-					execute: async ({ generatedTask, contentToAdd }) => {
-						task = generatedTask;
-						thingToAdd = contentToAdd?.thing;
-						addContent = contentToAdd?.content;
-					},
-				}),
-			},
-		});
-
-		console.log(task);
-
-		if ((task as string) === "add") {
-			// addString is the plaintext string that the user wants to add to the database
-			//chunk the note
-			let addString: string = addContent;
-			let vectorContent: Chunks = chunkNote(addContent);
-
-			if (thingToAdd === "page") {
-				// TODO: Sometimes this query hangs, and errors out. we need to do proper error management here.
-				const response = await fetch("https://md.dhr.wtf/?url=" + addContent, {
-					headers: {
-						Authorization: "Bearer " + c.env.SECURITY_KEY,
-					},
-				});
-
-				addString = await response.text();
-				vectorContent = chunkPage(addString);
-			}
-
-			// At this point, we can just go ahead and create the embeddings!
-			await batchCreateChunksAndEmbeddings({
-				store,
-				body: {
-					url: addContent,
-					user,
-					type: thingToAdd,
-					pageContent: addString,
-					title: `${addString.slice(0, 30)}... (Added from chatbot)`,
-				},
-				chunks: vectorContent,
-				env: c.env,
-			});
-
-			return c.json({
-				status: "ok",
-				response:
-					"I added the document to your personal second brain! You can now use it to answer questions or chat with me.",
-				contentAdded: {
-					type: thingToAdd,
-					content: addString,
-					url:
-						thingToAdd === "page"
-							? addContent
-							: `https://supermemory.ai/note/${Date.now()}`,
-				},
-			});
-		} else {
-			const filter: VectorizeVectorMetadataFilter = {
-				[`user-${user}`]: 1,
-			};
-
-			const queryAsVector = await store.embeddings.embedQuery(query);
-
-			const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, {
-				topK: 5,
-				filter,
-				returnMetadata: true,
-			});
-
-			const minScore = Math.min(...resp.matches.map(({ score }) => score));
-			const maxScore = Math.max(...resp.matches.map(({ score }) => score));
-
-			// This entire chat part is basically just a dumb down version of the /api/chat endpoint.
-			const normalizedData = resp.matches.map((data) => ({
-				...data,
-				normalizedScore:
-					maxScore !== minScore
-						? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98
-						: 50,
-			}));
-
-			const preparedContext = normalizedData.map(
-				({ metadata, score, normalizedScore }) => ({
-					context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`,
-					score,
-					normalizedScore,
-				}),
-			);
-
-			const prompt = template({
-				contexts: preparedContext,
-				question: query,
-			});
-
-			const initialMessages: CoreMessage[] = [
-				{
-					role: "system",
-					content: `You are an AI chatbot called "Supermemory.ai". When asked a question by a user, you must take all the context provided to you and give a good, small, but helpful response.`,
-				},
-				{ role: "assistant", content: "Hello, how can I help?" },
-			];
-
-			const userMessage: CoreMessage = { role: "user", content: prompt };
-
-			const response = await generateText({
-				model,
-				messages: [
-					...initialMessages,
-					...((chatHistory || []) as CoreMessage[]),
-					userMessage,
-				],
-			});
-
-			return c.json({ status: "ok", response: response.text });
-		}
-	},
-);
-
-/* TODO: Eventually, we should not have to save each user's content in a seperate vector.
-Lowkey, it makes sense. The user may save their own version of a page - like selected text from twitter.com url.
-But, it's not scalable *enough*. How can we store the same vectors for the same content, without needing to duplicate for each uer?
-Hard problem to solve, Vectorize doesn't have an OR filter, so we can't just filter by URL and user.
-*/
-app.post(
-	"/api/chat",
-	zValidator(
-		"query",
-		z.object({
-			query: z.string(),
-			user: z.string(),
-			topK: z.number().optional().default(10),
-			spaces: z.string().optional(),
-			sourcesOnly: z.string().optional().default("false"),
-			model: z.string().optional().default("gpt-4o"),
-			proMode: z.string().optional().default("false"),
-		}),
-	),
-	zValidator("json", chatObj),
-	async (c) => {
-		const query = c.req.valid("query");
-		const body = c.req.valid("json");
-
-		const sourcesOnly = query.sourcesOnly === "true";
-		const proMode = query.proMode === "true";
-
-		// Return early for dumb requests
-		if (sourcesOnly && body.sources) {
-			return c.json(body.sources);
-		}
-
-		const spaces = query.spaces?.split(",") ?? [undefined];
-
-		// Get the AI model maker and vector store
-		const { model, store } = await initQuery(c.env, query.model);
-
-		if (!body.sources) {
-			const filter: VectorizeVectorMetadataFilter = {
-				[`user-${query.user}`]: 1,
-			};
-
-			let proModeListedQueries: string[] = [];
-
-			if (proMode) {
-				const addedToQuery = (await c.env.AI.run(
-					// @ts-ignore
-					"@hf/nousresearch/hermes-2-pro-mistral-7b",
-					{
-						messages: [
-							{
-								role: "system",
-								content:
-									"You are a query enhancer. You must enhance a user's query to make it more relevant to what the user might be looking for. If there's any mention of dates like 'last summer' or 'this year', you should return 'DAY: X, MONTH: Y, YEAR: Z'. If there's any mention of locations, add that to the query too. Try to keep your responses as short as possible. Add to the user's query, don't replace it. Make sure to keep your answers short.",
-							},
-							{ role: "user", content: query.query },
-						],
-						tools: [
-							{
-								type: "function",
-								function: {
-									name: "Enhance query get list",
-									description:
-										"Enhance the user's query to make it more relevant",
-									parameters: {
-										type: "object",
-										properties: {
-											listedQueries: {
-												type: "array",
-												description: "List of queries that the user has asked",
-												items: {
-													type: "string",
-												},
-											},
-										},
-										required: ["Enhance query get list"],
-									},
-								},
-							},
-						],
-						max_tokens: 200,
-					},
-				)) as {
-					response?: string;
-					tool_calls?: {
-						name: string;
-						arguments: {
-							listedQueries: string[];
-						};
-					}[];
-				};
-
-				proModeListedQueries =
-					addedToQuery.tool_calls?.[0]?.arguments?.listedQueries ?? [];
-			}
-
-			// Converting the query to a vector so that we can search for similar vectors
-			const queryAsVector = await store.embeddings.embedQuery(
-				query.query + " " + proModeListedQueries.join(" "),
-			);
-			const responses: VectorizeMatches = { matches: [], count: 0 };
-
-			// SLICED to 5 to avoid too many queries
-			for (const space of spaces.slice(0, 5)) {
-				if (space && space.length >= 1) {
-					// it's possible for space list to be [undefined] so we only add space filter conditionally
-					filter[`space-${query.user}-${space}`] = 1;
-				}
-
-				// Because there's no OR operator in the filter, we have to make multiple queries
-				const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, {
-					topK: query.topK,
-					filter,
-					returnMetadata: true,
-				});
-
-				// Basically recreating the response object
-				if (resp.count > 0) {
-					responses.matches.push(...resp.matches);
-					responses.count += resp.count;
-				}
-			}
-
-			const minScore = Math.min(...responses.matches.map(({ score }) => score));
-			const maxScore = Math.max(...responses.matches.map(({ score }) => score));
-
-			// We are "normalising" the scores - if all of them are on top, we want to make sure that
-			// we have a way to filter out the noise.
-			const normalizedData = responses.matches.map((data) => ({
-				...data,
-				normalizedScore:
-					maxScore !== minScore
-						? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98
-						: 50, // If all scores are the same, set them to the middle of the scale
-			}));
-
-			let highScoreData = normalizedData.filter(
-				({ normalizedScore }) => normalizedScore > 50,
-			);
-
-			// If the normalsation is not done properly, we have a fallback to just get the
-			// top 3 scores
-			if (highScoreData.length === 0) {
-				highScoreData = normalizedData
-					.sort((a, b) => b.score - a.score)
-					.slice(0, 3);
-			}
-
-			const sortedHighScoreData = highScoreData.sort(
-				(a, b) => b.normalizedScore - a.normalizedScore,
-			);
-
-			body.sources = {
-				normalizedData,
-			};
-
-			// So this is kinda hacky, but the frontend needs to do 2 calls to get sources and chat.
-			// I think this is fine for now, but we can improve this later.
-			if (sourcesOnly) {
-				const idsAsStrings = sortedHighScoreData.map((dataPoint) =>
-					dataPoint.id.toString(),
-				);
-
-				const storedContent = await Promise.all(
-					idsAsStrings.map(async (id) => await c.env.KV.get(id)),
-				);
-
-				const metadata = normalizedData.map((datapoint) => datapoint.metadata);
-				return c.json({
-					ids: storedContent.filter(Boolean),
-					metadata,
-					normalizedData,
-					proModeListedQueries,
-				});
-			}
-		}
-
-		//Serach mem0
-		type Mem0Response = {
-			id: string;
-			memory: string;
-			user_id: string;
-			hash: string;
-			metadata: any;
-			categories: any;
-			created_at: string;
-			updated_at: string;
-		};
-		const mem0Response = await fetch(
-			"https://api.mem0.ai/v1/memories/?user_id=" + query.user,
-			{
-				method: "GET",
-				headers: {
-					Authorization: `Token ${c.env.MEM0_API_KEY}`,
-				},
-			},
-		);
-		const contextFromMem0 = await mem0Response.text();
-		const contextJson: Mem0Response[] = await JSON.parse(contextFromMem0);
-		const memories = contextJson.map((item) => {
-			return item.memory;
-		});
-		console.log("Here are the mem0 memories", memories);
-
-		const preparedContext = body.sources.normalizedData.map(
-			({ metadata, score, normalizedScore }) => ({
-				context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`,
-				score,
-				normalizedScore,
-			}),
-		);
-
-		const initialMessages: CoreMessage[] = [
-			{ role: "user", content: systemPrompt },
-			{
-				role: "assistant",
-				content: `Here is the profile of the user, refer this whenever possible ${memories.join(", ")}}`,
-			}, // prase and add memory json here
-		];
-
-		const prompt = template({
-			contexts: preparedContext,
-			question: query.query,
-		});
-
-		const userMessage: CoreMessage = { role: "user", content: prompt };
-
-		const response = await streamText({
-			model: model,
-			messages: [
-				...initialMessages,
-				...((body.chatHistory || []) as CoreMessage[]),
-				userMessage,
-			],
-			// temperature: 0.4,
-		});
-
-		return response.toTextStreamResponse();
-	},
-);
-
-app.post(
-	"/api/delete",
-	zValidator(
-		"json",
-		z.object({
-			websiteUrl: z.string(),
-			user: z.string(),
-		}),
-	),
-	async (c) => {
-		const { websiteUrl, user } = c.req.valid("json");
-
-		const { store } = await initQuery(c.env);
-
-		await deleteDocument({ url: websiteUrl, user, c, store });
-
-		return c.json({ message: "Document deleted" });
-	},
-);
-
-// ERROR #1 - this is the api that the editor uses, it is just a scrape off of /api/chat so you may check that out
-app.get(
-	"/api/editorai",
-	zValidator(
-		"query",
-		z.object({
-			context: z.string(),
-			request: z.string(),
-		}),
-	),
-	async (c) => {
-		const { context, request } = c.req.valid("query");
-		const { model } = await initQuery(c.env);
-
-		const response = await streamText({
-			model,
-			prompt: `${request}-${context}`,
-			maxTokens: 224,
-		});
-
-		return response.toTextStreamResponse();
-	},
-);
-
-app.get("/howFuckedAreWe", async (c) => {
-	let keys = 0;
-	const concurrencyLimit = 5; // Adjust this based on your system's capability
-	const queue: string[] = [undefined]; // Start with an undefined cursor
-
-	async function fetchKeys(cursor?: string): Promise<void> {
-		const response = await c.env.KV.list({ cursor });
-		keys += response.keys.length;
-
-		// @ts-ignore
-		if (response.cursor) {
-			// @ts-ignore
-			queue.push(response.cursor);
-		}
-	}
-
-	async function getAllKeys(): Promise<void> {
-		const promises: Promise<void>[] = [];
-
-		while (queue.length > 0) {
-			while (promises.length < concurrencyLimit && queue.length > 0) {
-				const cursor = queue.shift();
-				promises.push(fetchKeys(cursor));
-			}
-
-			await Promise.all(promises);
-			promises.length = 0; // Clear the promises array
-		}
-	}
-
-	await getAllKeys();
-
-	console.log(`Total number of keys: ${keys}`);
-
-	// on a scale of 200,000
-	// what % are we there?
-	const fuckedPercent = (keys / 200000) * 100;
-
-	return c.json({ fuckedPercent });
-});
-
-export default {
-	fetch: app.fetch,
-	queue,
-};

apps/cf-ai-backend/src/prompts/prompt1.ts

@@ -1,42 +0,0 @@
-export const systemPrompt = `You are an AI assistant called Supermemory that acts as a "Second Brain" by answering questions based on provided context. Your goal is to directly address the question concisely and to the point, without excessive elaboration.
-
-Multiple pieces of context, each with an associated relevance score, will be provided. Each context piece and its score will be enclosed within the following tags: <context> and <context_score>. The question you need to answer will be enclosed within the <question> tags.
-
-To generate your answer:
-- Carefully analyze the question and identify the key information needed to address it
-- Locate the specific parts of each context that contain this key information
-- Compare the relevance scores of the provided contexts
-- Concisely summarize the relevant information from the higher-scoring context(s) in your own words
-- Provide a direct answer to the question
-- Use markdown formatting in your answer, including bold, italics, and bullet points as appropriate to improve readability and highlight key points
-- Give detailed and accurate responses for things like 'write a blog' or long-form questions.
-- The normalisedScore is a value in which the scores are 'balanced' to give a better representation of the relevance of the context, between 1 and 100, out of the top 10 results
-- provide your justification in the end, in a <justification> </justification> tag
-If no context is provided, introduce yourself and explain that the user can save content which will allow you to answer questions about that content in the future. Do not provide an answer if no context is provided.`;
-
-export const template = ({ contexts, question }) => {
-	// Map over contexts to generate the context and score parts
-	const contextParts = contexts
-		.map(
-			({ context, normalisedScore }) => `
-  
-  ${context ? `<context> ${context} </context>` : ""}
-  
-  ${normalisedScore ? `<context_score> normalisedScore: ${normalisedScore} </context_score>` : ""}`,
-		)
-		.join("\n");
-
-	// Construct the final prompt using a template literal
-	const finalPrompt = `
-  Here's the given context and question for the task:
-  ${contextParts}
-  
-  The question is provided in the prompt below:
-  
-  <question>
-  ${question}
-  </question>
-  `;
-
-	return finalPrompt.trim();
-};

apps/cf-ai-backend/src/queueConsumer/chunkers/chonker.ts

@@ -1,47 +0,0 @@
-import nlp from "compromise";
-
-/**
- * Split text into chunks of specified max size with some overlap for continuity.
- */
-export default function chunkText(
-	text: string,
-	maxChunkSize: number,
-	overlap: number = 0.2,
-): string[] {
-	const sentences = nlp(text).sentences().out("array");
-	const chunks = [];
-	let currentChunk: string[] = [];
-	let currentSize = 0;
-
-	for (let i = 0; i < sentences.length; i++) {
-		const sentence = sentences[i];
-		currentChunk.push(sentence);
-		currentSize += sentence.length;
-
-		if (currentSize >= maxChunkSize) {
-			// Calculate overlap
-			const overlapSize = Math.floor(currentChunk.length * overlap);
-			const chunkText = currentChunk.join(" ");
-			chunks.push({
-				text: chunkText,
-				start: i - currentChunk.length + 1,
-				end: i,
-			});
-
-			// Prepare the next chunk with overlap
-			currentChunk = currentChunk.slice(-overlapSize);
-			currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0);
-		}
-	}
-
-	if (currentChunk.length > 0) {
-		const chunkText = currentChunk.join(" ");
-		chunks.push({
-			text: chunkText,
-			start: sentences.length - currentChunk.length,
-			end: sentences.length,
-		});
-	}
-
-	return chunks.map((chunk) => chunk.text);
-}

apps/cf-ai-backend/src/queueConsumer/chunkers/chunkPageOrNotes.ts

@@ -1,13 +0,0 @@
-import chunkText from "./chonker";
-import { PageOrNoteChunks } from "../../types";
-export function chunkPage(pageContent: string): PageOrNoteChunks {
-	const chunks = chunkText(pageContent, 1536);
-
-	return { type: "page", chunks: chunks };
-}
-
-export function chunkNote(noteContent: string): PageOrNoteChunks {
-	const chunks = chunkText(noteContent, 1536);
-
-	return { type: "note", chunks: chunks };
-}

apps/cf-ai-backend/src/queueConsumer/chunkers/chunkTweet.ts

@@ -1,75 +0,0 @@
-import { TweetChunks } from "../../types";
-import chunkText from "./chonker";
-import { getRawTweet } from "@repo/shared-types/utils";
-
-interface Tweet {
-	id: string;
-	text: string;
-	links: Array<string>;
-	images: Array<string>;
-	videos: Array<string>;
-}
-interface Metadata {
-	tweetId: string;
-	tweetLinks: any[];
-	tweetVids: any[];
-	tweetImages: any[];
-}
-
-export interface ThreadTweetData {
-	chunkedTweet: string[];
-	metadata: Metadata;
-}
-
-export function chunkThread(threadText: string): TweetChunks {
-	let thread = threadText;
-
-	try {
-		thread = JSON.parse(threadText);
-	} catch (e) {
-		console.log("error: thread is not json.", e);
-	}
-
-	if (typeof threadText == "string") {
-		console.log("DA WORKER FAILED DO SOMEHTING FIX DA WROKER", thread);
-		const rawTweet = getRawTweet(thread);
-		console.log(rawTweet);
-		const parsedTweet: any = JSON.parse(rawTweet);
-
-		const chunkedTweet = chunkText(parsedTweet.text, 1536);
-		const metadata: Metadata = {
-			tweetId: parsedTweet.id_str,
-			tweetLinks: parsedTweet.entities?.urls.map(
-				(url: any) => url.expanded_url,
-			),
-			tweetVids:
-				parsedTweet.extended_entities?.media
-					.filter((media: any) => media.type === "video")
-					.map((media: any) => media.video_info!.variants[0].url) || [],
-			tweetImages:
-				parsedTweet.extended_entities?.media
-					.filter((media: any) => media.type === "photo")
-					.map((media: any) => media.media_url_https!) || [],
-		};
-
-		const chunks = [{ chunkedTweet: chunkedTweet, metadata }];
-
-		return { type: "tweet", chunks };
-	} else {
-		console.log("thread in else statement", JSON.stringify(thread));
-		const chunkedTweets = (thread as any).map((tweet: Tweet) => {
-			const chunkedTweet = chunkText(tweet.text, 1536);
-
-			const metadata = {
-				tweetId: tweet.id,
-				tweetLinks: tweet.links,
-				tweetVids: tweet.videos,
-				tweetImages: tweet.images,
-			};
-
-			return { chunkedTweet, metadata };
-		});
-
-		return { type: "tweet", chunks: chunkedTweets };
-	}
-}

apps/cf-ai-backend/src/queueConsumer/helpers/processNotes.ts

@@ -1,36 +0,0 @@
-import { Result, Ok, Err } from "../../errors/results";
-import { BaseError } from "../../errors/baseError";
-import { Metadata } from "../utils/get-metadata";
-
-class ProcessNotesError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[Note Processing Error]", message, source);
-	}
-}
-
-type ProcessNoteResult = {
-	noteContent: { noteId: number; noteContent: string };
-	metadata: Metadata;
-};
-
-export function processNote(
-	content: string,
-): Result<ProcessNoteResult, ProcessNotesError> {
-	try {
-		const pageContent = content;
-		const noteId = new Date().getTime();
-
-		const metadata = {
-			baseUrl: `https://supermemory.ai/note/${noteId}`,
-			description: `Note created at ${new Date().toLocaleString()}`,
-			image: "https://supermemory.ai/logo.png",
-			title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`,
-		};
-
-		const noteContent = { noteId: noteId, noteContent: pageContent };
-		return Ok({ noteContent, metadata });
-	} catch (e) {
-		console.error("[Note Processing Error]", e);
-		return Err(new ProcessNotesError((e as Error).message, "processNote"));
-	}
-}

apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts

@@ -1,43 +0,0 @@
-import { Result, Ok, Err, isErr } from "../../errors/results";
-import { BaseError } from "../../errors/baseError";
-import { getMetaData, Metadata } from "../utils/get-metadata";
-
-class ProcessPageError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[Page Proceessing Error]", message, source);
-	}
-}
-
-type PageProcessResult = { pageContent: string; metadata: Metadata };
-
-export async function processPage(input: {
-	url: string;
-	securityKey: string;
-}): Promise<Result<PageProcessResult, ProcessPageError>> {
-	try {
-		const response = await fetch("https://md.dhr.wtf/?url=" + input.url, {
-			headers: {
-				Authorization: "Bearer " + input.securityKey,
-			},
-		});
-		const pageContent = await response.text();
-		if (!pageContent) {
-			return Err(
-				new ProcessPageError(
-					"Failed to get response form markdowner",
-					"processPage",
-				),
-			);
-		}
-		const metadataResult = await getMetaData(input.url);
-		if (isErr(metadataResult)) {
-			throw metadataResult.error;
-		}
-		const metadata = metadataResult.value;
-		console.log("[this is the metadata]", metadata);
-		return Ok({ pageContent, metadata });
-	} catch (e) {
-		console.error("[Page Processing Error]", e);
-		return Err(new ProcessPageError((e as Error).message, "processPage"));
-	}
-}

apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts

@@ -1,88 +0,0 @@
-import { Tweet } from "react-tweet/api";
-import { Result, Ok, Err, isErr } from "../../errors/results";
-import { BaseError } from "../../errors/baseError";
-import { getMetaData, Metadata } from "../utils/get-metadata";
-import { tweetToMd } from "@repo/shared-types/utils"; // can I do this?
-import { Env } from "../../types";
-
-class ProcessTweetError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[Tweet Proceessing Error]", message, source);
-	}
-}
-
-type GetTweetResult = Tweet;
-
-export const getTweetData = async (
-	tweetID: string,
-): Promise<Result<GetTweetResult, ProcessTweetError>> => {
-	try {
-		console.log("is fetch defined here?");
-		const url = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetID}&lang=en&features=tfw_timeline_list%3A%3Btfw_follower_count_sunset%3Atrue%3Btfw_tweet_edit_backend%3Aon%3Btfw_refsrc_session%3Aon%3Btfw_fosnr_soft_interventions_enabled%3Aon%3Btfw_show_birdwatch_pivots_enabled%3Aon%3Btfw_show_business_verified_badge%3Aon%3Btfw_duplicate_scribes_to_settings%3Aon%3Btfw_use_profile_image_shape_enabled%3Aon%3Btfw_show_blue_verified_badge%3Aon%3Btfw_legacy_timeline_sunset%3Atrue%3Btfw_show_gov_verified_badge%3Aon%3Btfw_show_business_affiliate_badge%3Aon%3Btfw_tweet_edit_frontend%3Aon&token=4c2mmul6mnh`;
-
-		const resp = await fetch(url, {
-			headers: {
-				"User-Agent":
-					"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
-				Accept: "application/json",
-				"Accept-Language": "en-US,en;q=0.5",
-				"Accept-Encoding": "gzip, deflate, br",
-				Connection: "keep-alive",
-				"Upgrade-Insecure-Requests": "1",
-				"Cache-Control": "max-age=0",
-				TE: "Trailers",
-			},
-		});
-		console.log(resp.status);
-
-		const data = (await resp.json()) as Tweet;
-
-		return Ok(data);
-	} catch (e) {
-		console.error("[Tweet Proceessing Error]", e);
-		return Err(new ProcessTweetError(e, "getTweetData"));
-	}
-};
-
-export const getThreadData = async (input: {
-	tweetUrl: string;
-	env: Env;
-}): Promise<Result<any, ProcessTweetError>> => {
-	try {
-		// const threadRequest = await fetch(input.cf_thread_endpoint, {
-		// 	method: "POST",
-		// 	headers: {
-		// 		"Content-Type": "application/json",
-		// 		Authorization: input.authKey,
-		// 	},
-		// 	body: JSON.stringify({ url: input.tweetUrl }),
-		// });
-		// if (threadRequest.status !== 200) {
-		// 	console.log(await threadRequest.text());
-		// 	console.log(input.tweetUrl);
-		// 	return Err(
-		// 		new ProcessTweetError(
-		// 			`Failed to fetch the thread: ${input.tweetUrl}, Reason: ${threadRequest.statusText}`,
-		// 			"getThreadData",
-		// 		),
-		// 	);
-		// }
-		//@ts-ignore
-		const thread = await input.env.THREAD.processTweets(input.tweetUrl);
-		console.log("[thread response]", thread);
-
-		if (!thread.length) {
-			console.log("Thread is an empty array");
-			return Err(
-				new ProcessTweetError(
-					"[THREAD FETCHING SERVICE] Got no content form thread worker",
-					"getThreadData",
-				),
-			);
-		}
-		return Ok(thread);
-	} catch (e) {
-		console.error("[Thread Processing Error]", e);
-		return Err(new ProcessTweetError((e as Error).message, "getThreadData"));
-	}
-};

apps/cf-ai-backend/src/queueConsumer/index.ts

@@ -1,379 +0,0 @@
-import { Env, PageOrNoteChunks, TweetChunks, vectorObj } from "../types";
-import { typeDecider } from "./utils/typeDecider";
-import { isErr, wrap } from "../errors/results";
-import { processNote } from "./helpers/processNotes";
-import { processPage } from "./helpers/processPage";
-import { getThreadData, getTweetData } from "./helpers/processTweet";
-import { tweetToMd } from "@repo/shared-types/utils";
-import { chunkNote, chunkPage } from "./chunkers/chunkPageOrNotes";
-import { chunkThread } from "./chunkers/chunkTweet";
-import { batchCreateChunksAndEmbeddings, initQuery } from "../helper";
-import { z } from "zod";
-import { Metadata } from "./utils/get-metadata";
-import { BaseError } from "../errors/baseError";
-import { database } from "../db";
-import {
-	storedContent,
-	space,
-	contentToSpace,
-	users,
-	jobs,
-	Job,
-} from "@repo/db/schema";
-import { and, eq, inArray, sql } from "drizzle-orm";
-
-class VectorInsertError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[Vector Insert Error]", message, source);
-	}
-}
-const vectorErrorFactory = (err: Error) => new VectorInsertError(err.message);
-
-class D1InsertError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[D1 Insert Error]", message, source);
-	}
-}
-
-const d1ErrorFactory = (err: Error, source: string) =>
-	new D1InsertError(err.message, source);
-
-const calculateExponentialBackoff = (
-	attempts: number,
-	baseDelaySeconds: number,
-) => {
-	return baseDelaySeconds ** attempts;
-};
-
-const BASE_DELAY_SECONDS = 5;
-export async function queue(
-	batch: MessageBatch<{
-		content: string;
-		space: Array<number>;
-		user: string;
-		type: string;
-	}>,
-	env: Env,
-): Promise<void> {
-	const db = database(env);
-
-	for (let message of batch.messages) {
-		const body = message.body;
-
-		const type = body.type;
-		const userExists = await wrap(
-			db.select().from(users).where(eq(users.id, body.user)).limit(1),
-			d1ErrorFactory,
-			"Error when trying to verify user",
-		);
-
-		if (isErr(userExists)) {
-			throw userExists.error;
-		}
-
-		//check if this is a retry job.. by checking if the combination of the userId and the url already exists on the queue
-		let jobId;
-		const existingJob = await wrap(
-			db
-				.select()
-				.from(jobs)
-				.where(
-					and(
-						eq(jobs.userId, userExists.value[0].id),
-						eq(jobs.url, body.content),
-					),
-				)
-				.limit(1),
-			d1ErrorFactory,
-			"Error when checking for existing job",
-		);
-
-		if (isErr(existingJob)) {
-			throw existingJob.error;
-		}
-
-		if (existingJob.value.length > 0) {
-			jobId = existingJob.value[0].id;
-			await wrap(
-				db
-					.update(jobs)
-					.set({
-						attempts: existingJob.value[0].attempts + 1,
-						updatedAt: new Date(),
-						status: "Processing",
-					})
-					.where(eq(jobs.id, jobId)),
-				d1ErrorFactory,
-				"Error when updating job attempts",
-			);
-		} else {
-			const job = await wrap(
-				db
-					.insert(jobs)
-					.values({
-						userId: userExists.value[0].id as string,
-						url: body.content,
-						status: "Processing",
-						attempts: 1,
-						createdAt: new Date(),
-						updatedAt: new Date(),
-					})
-					.returning({ jobId: jobs.id }),
-				d1ErrorFactory,
-				"Error When inserting into jobs table",
-			);
-			if (isErr(job)) {
-				throw job.error;
-			}
-			jobId = job.value[0].jobId;
-		}
-
-		let pageContent: string;
-		let vectorData: string;
-		let metadata: Metadata;
-		let storeToSpaces = body.space;
-		let chunks: TweetChunks | PageOrNoteChunks;
-		let noteId = 0;
-		switch (type) {
-			case "note": {
-				console.log("note hit");
-				const note = processNote(body.content);
-				if (isErr(note)) {
-					throw note.error;
-				}
-				pageContent = note.value.noteContent.noteContent;
-				noteId = note.value.noteContent.noteId;
-				metadata = note.value.metadata;
-				vectorData = pageContent;
-				chunks = chunkNote(pageContent);
-				break;
-			}
-			case "page": {
-				console.log("page hit");
-				const page = await processPage({
-					url: body.content,
-					securityKey: env.MD_SEC_KEY,
-				});
-				if (isErr(page)) {
-					console.log("there is a page error here");
-					throw page.error;
-				}
-				pageContent = page.value.pageContent;
-				metadata = page.value.metadata;
-				vectorData = pageContent;
-				chunks = chunkPage(pageContent);
-				break;
-			}
-
-			case "tweet": {
-				const tweet = await getTweetData(body.content.split("/").pop());
-
-				const thread = await getThreadData({
-					tweetUrl: body.content,
-					env: env,
-				});
-				console.log("[This is the thread]", thread);
-				if (isErr(tweet)) {
-					throw tweet.error;
-				}
-				pageContent = tweetToMd(tweet.value);
-
-				metadata = {
-					baseUrl: body.content,
-					description: tweet.value.text.slice(0, 200),
-					image: tweet.value.user.profile_image_url_https,
-					title: `Tweet by ${tweet.value.user.name}`,
-				};
-
-				if (isErr(thread)) {
-					console.log("Thread worker is down!");
-					vectorData = JSON.stringify(pageContent);
-					console.error(thread.error);
-				} else {
-					console.log("thread worker is fine");
-					vectorData = thread.value;
-				}
-				chunks = chunkThread(vectorData);
-				break;
-			}
-		}
-
-		//add to mem0, abstract
-
-		const includeMessages = {
-			note: "information of this user based on the provided note.",
-			tweet: "interests of this user based on a twitter post they are saving.",
-			page: "interests of this user based on a web page they are saving.",
-		};
-
-		const mem0Response = fetch("https://api.mem0.ai/v1/memories/", {
-			method: "POST",
-			headers: {
-				"Content-Type": "application/json",
-				Authorization: `Token ${env.MEM0_API_KEY}`,
-			},
-			body: JSON.stringify({
-				messages: [
-					{
-						role: "user",
-						content: `Extract information about the user based on this saved content provided, remember that the date was ${new Date().toUTCString()} in utc time zone`,
-					},
-					{
-						role: "user",
-						content: vectorData.replace(/<raw>.*?<\/raw>/g, ""),
-					},
-				],
-				includes: includeMessages[type],
-				user_id: body.user,
-			}),
-		});
-		// see what's up with the storedToSpaces in this block
-		const { store } = await initQuery(env);
-
-		type body = z.infer<typeof vectorObj>;
-
-		const Chunkbody: body = {
-			pageContent: pageContent,
-			spaces: storeToSpaces.map((spaceId) => spaceId.toString()),
-			user: body.user,
-			type: type,
-			url: metadata.baseUrl,
-			description: metadata.description,
-			title: metadata.title,
-		};
-
-		try {
-			const vectorResult = await wrap(
-				batchCreateChunksAndEmbeddings({
-					store: store,
-					body: Chunkbody,
-					chunks: chunks,
-					env: env,
-				}),
-				vectorErrorFactory,
-				"Error when Inserting into vector database",
-			);
-
-			if (isErr(vectorResult)) {
-				await db
-					.update(jobs)
-					.set({ error: vectorResult.error.message, status: "error" })
-					.where(eq(jobs.id, jobId));
-				message.retry({
-					delaySeconds: calculateExponentialBackoff(
-						message.attempts,
-						BASE_DELAY_SECONDS,
-					),
-				});
-				throw vectorResult.error;
-			}
-
-			const saveToDbUrl =
-				(metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) +
-				"#supermemory-user-" +
-				body.user;
-
-			let contentId: number;
-
-			const insertResponse = await wrap(
-				db
-					.insert(storedContent)
-					.values({
-						content: pageContent as string,
-						title: metadata.title,
-						description: metadata.description,
-						url: saveToDbUrl,
-						baseUrl: saveToDbUrl,
-						image: metadata.image,
-						savedAt: new Date(),
-						userId: body.user,
-						type: type,
-						noteId: noteId,
-					})
-					.returning({ id: storedContent.id }),
-				d1ErrorFactory,
-				"Error when inserting into storedContent",
-			);
-
-			if (isErr(insertResponse)) {
-				await db
-					.update(jobs)
-					.set({ error: insertResponse.error.message, status: "error" })
-					.where(eq(jobs.id, jobId));
-				message.retry({
-					delaySeconds: calculateExponentialBackoff(
-						message.attempts,
-						BASE_DELAY_SECONDS,
-					),
-				});
-				throw insertResponse.error;
-			}
-
-			contentId = insertResponse.value[0].id;
-			if (storeToSpaces.length > 0) {
-				// Adding the many-to-many relationship between content and spaces
-				const spaceData = await wrap(
-					db
-						.select()
-						.from(space)
-						.where(
-							and(inArray(space.id, storeToSpaces), eq(space.user, body.user)),
-						)
-						.all(),
-					d1ErrorFactory,
-					"Error when getting data from spaces",
-				);
-
-				if (isErr(spaceData)) {
-					throw spaceData.error;
-				}
-				try {
-					await Promise.all(
-						spaceData.value.map(async (s) => {
-							try {
-								await db
-									.insert(contentToSpace)
-									.values({ contentId: contentId, spaceId: s.id });
-
-								await db.update(space).set({ numItems: s.numItems + 1 });
-							} catch (e) {
-								console.error(`Error updating space ${s.id}:`, e);
-								throw e;
-							}
-						}),
-					);
-				} catch (e) {
-					console.error("Error in updateSpacesWithContent:", e);
-					throw new Error(`Failed to update spaces: ${e.message}`);
-				}
-			}
-		} catch (e) {
-			console.error("Error in simulated transaction", e.message);
-
-			message.retry({
-				delaySeconds: calculateExponentialBackoff(
-					message.attempts,
-					BASE_DELAY_SECONDS,
-				),
-			});
-			throw new D1InsertError(
-				"Error when inserting into d1",
-				"D1 stuff after the vectorize",
-			);
-		}
-
-		// After the d1 and vectories suceeds then finally update the jobs table to indicate that the job has completed
-
-		await db
-			.update(jobs)
-			.set({ status: "Processed" })
-			.where(eq(jobs.id, jobId));
-	}
-	console.log(`Finished processing batch of ${batch.messages.length} length`);
-}
-
-/*
-To do: 
-Figure out rate limits!!
-
-*/

apps/cf-ai-backend/src/queueConsumer/utils/get-metadata.ts

@@ -1,57 +0,0 @@
-import * as cheerio from "cheerio";
-import { Result, Ok, Err } from "../../errors/results";
-import { BaseError } from "../../errors/baseError";
-
-class GetMetadataError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[Fetch Metadata Error]", message, source);
-	}
-}
-export type Metadata = {
-	title: string;
-	description: string;
-	image: string;
-	baseUrl: string;
-};
-// TODO: THIS SHOULD PROBABLY ALSO FETCH THE OG-IMAGE
-export async function getMetaData(
-	url: string,
-): Promise<Result<Metadata, GetMetadataError>> {
-	try {
-		const response = await fetch(url);
-		const html = await response.text();
-
-		const $ = cheerio.load(html);
-
-		// Extract the base URL
-		const baseUrl = url;
-
-		// Extract title
-		const title = $("title").text().trim();
-
-		const description = $("meta[name=description]").attr("content") ?? "";
-
-		const _favicon =
-			$("link[rel=icon]").attr("href") ?? "https://supermemory.dhr.wtf/web.svg";
-
-		let favicon =
-			_favicon.trim().length > 0
-				? _favicon.trim()
-				: "https://supermemory.dhr.wtf/web.svg";
-		if (favicon.startsWith("/")) {
-			favicon = baseUrl + favicon;
-		} else if (favicon.startsWith("./")) {
-			favicon = baseUrl + favicon.slice(1);
-		}
-
-		return Ok({
-			title,
-			description,
-			image: favicon,
-			baseUrl,
-		});
-	} catch (e) {
-		console.error("[Metadata Fetch Error]", e);
-		return Err(new GetMetadataError((e as Error).message, "getMetaData"));
-	}
-}

apps/cf-ai-backend/src/queueConsumer/utils/typeDecider.ts

@@ -1,34 +0,0 @@
-import { Result, Ok, Err } from "../../errors/results";
-import { BaseError } from "../../errors/baseError";
-
-export type contentType = "page" | "tweet" | "note";
-
-class GetTypeError extends BaseError {
-	constructor(message?: string, source?: string) {
-		super("[Decide Type Error]", message, source);
-	}
-}
-export const typeDecider = (
-	content: string,
-): Result<contentType, GetTypeError> => {
-	try {
-		// if the content is a URL, then it's a page. if its a URL with https://x.com/user/status/123, then it's a tweet. else, it's a note.
-		// do strict checking with regex
-		if (
-			content.match(/https?:\/\/(x\.com|twitter\.com)\/[\w]+\/[\w]+\/[\d]+/)
-		) {
-			return Ok("tweet");
-		} else if (
-			content.match(
-				/^(https?:\/\/)?(www\.)?[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,5}(\/.*)?$/i,
-			)
-		) {
-			return Ok("page");
-		} else {
-			return Ok("note");
-		}
-	} catch (e) {
-		console.error("[Decide Type Error]", e);
-		return Err(new GetTypeError((e as Error).message, "typeDecider"));
-	}
-};