mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-10-06 02:47:54 +00:00
auto generated openapi schema
This commit is contained in:
Dhravya Shah
parent
8f016b3bfa
commit
ff14d77c11
9 changed files with 404 additions and 353 deletions
|
|
@ -24,6 +24,7 @@
|
|||
"openai": "^4.68.4",
|
||||
"postgres": "^3.4.4",
|
||||
"uuid": "^11.0.1",
|
||||
"wrangler": "^3.111.0",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import spacesRoute from "./routes/spaces";
|
|||
import actions from "./routes/actions";
|
||||
import memories from "./routes/memories";
|
||||
import integrations from "./routes/integrations";
|
||||
import { fromHono, OpenAPIRoute } from "chanfana";
|
||||
import {
|
||||
cloudflareRateLimiter,
|
||||
DurableObjectRateLimiter,
|
||||
|
|
@ -24,190 +25,203 @@ import {
|
|||
} from "@hono-rate-limiter/cloudflare";
|
||||
import { ConfigType, GeneralConfigType, rateLimiter } from "hono-rate-limiter";
|
||||
|
||||
export const app = new Hono<{ Variables: Variables; Bindings: Env }>()
|
||||
.use("*", timing())
|
||||
.use("*", logger())
|
||||
.use(
|
||||
"*",
|
||||
cors({
|
||||
origin: [
|
||||
"http://localhost:3000",
|
||||
"https://supermemory.ai",
|
||||
"https://*.supermemory.ai",
|
||||
"https://*.supermemory.com",
|
||||
"https://supermemory.com",
|
||||
"chrome-extension://*",
|
||||
],
|
||||
allowHeaders: ["*"],
|
||||
allowMethods: ["*"],
|
||||
credentials: true,
|
||||
exposeHeaders: ["*"],
|
||||
})
|
||||
)
|
||||
.use("/v1/*", auth)
|
||||
.use("/v1/*", (c, next) => {
|
||||
const user = c.get("user");
|
||||
|
||||
if (c.env.NODE_ENV === "development") {
|
||||
return next();
|
||||
}
|
||||
// Create base Hono app first
|
||||
const honoApp = new Hono<{ Variables: Variables; Bindings: Env }>();
|
||||
|
||||
// RATELIMITS
|
||||
const rateLimitConfig = {
|
||||
// Endpoints that bypass rate limiting
|
||||
excludedPaths: [
|
||||
"/v1/add",
|
||||
"/v1/chat",
|
||||
"/v1/suggested-learnings",
|
||||
"/v1/recommended-questions",
|
||||
] as (string | RegExp)[],
|
||||
const app = fromHono(honoApp, {
|
||||
base: "",
|
||||
});
|
||||
|
||||
// Custom rate limits for specific endpoints
|
||||
customLimits: {
|
||||
notionImport: {
|
||||
paths: ["/v1/integrations/notion/import", "/v1/integrations/notion"],
|
||||
windowMs: 10 * 60 * 1000, // 10 minutes
|
||||
limit: 5, // 5 requests per 10 minutes
|
||||
},
|
||||
inviteSpace: {
|
||||
paths: [/^\v1\/spaces\/[^/]+\/invite$/],
|
||||
windowMs: 60 * 1000, // 1 minute
|
||||
limit: 5, // 5 requests per minute
|
||||
},
|
||||
} as Record<
|
||||
string,
|
||||
{ paths: (string | RegExp)[]; windowMs: number; limit: number }
|
||||
>,
|
||||
// Add all middleware and routes
|
||||
app.use("*", timing());
|
||||
app.use("*", logger());
|
||||
app.use(
|
||||
"*",
|
||||
cors({
|
||||
origin: [
|
||||
"http://localhost:3000",
|
||||
"https://supermemory.ai",
|
||||
"https://*.supermemory.ai",
|
||||
"https://*.supermemory.com",
|
||||
"https://supermemory.com",
|
||||
"chrome-extension://*",
|
||||
],
|
||||
allowHeaders: ["*"],
|
||||
allowMethods: ["*"],
|
||||
credentials: true,
|
||||
exposeHeaders: ["*"],
|
||||
})
|
||||
);
|
||||
|
||||
default: {
|
||||
windowMs: 60 * 1000, // 1 minute
|
||||
limit: 100, // 100 requests per minute
|
||||
app.use("/v1/*", auth);
|
||||
app.use("/v1/*", (c, next) => {
|
||||
const user = c.get("user");
|
||||
|
||||
if (c.env.NODE_ENV === "development") {
|
||||
return next();
|
||||
}
|
||||
|
||||
// RATELIMITS
|
||||
const rateLimitConfig = {
|
||||
// Endpoints that bypass rate limiting
|
||||
excludedPaths: [
|
||||
"/v1/add",
|
||||
"/v1/chat",
|
||||
"/v1/suggested-learnings",
|
||||
"/v1/recommended-questions",
|
||||
] as (string | RegExp)[],
|
||||
|
||||
// Custom rate limits for specific endpoints
|
||||
customLimits: {
|
||||
notionImport: {
|
||||
paths: ["/v1/integrations/notion/import", "/v1/integrations/notion"],
|
||||
windowMs: 10 * 60 * 1000, // 10 minutes
|
||||
limit: 5, // 5 requests per 10 minutes
|
||||
},
|
||||
inviteSpace: {
|
||||
paths: [/^\v1\/spaces\/[^/]+\/invite$/],
|
||||
windowMs: 60 * 1000, // 1 minute
|
||||
limit: 5, // 5 requests per minute
|
||||
},
|
||||
} as Record<
|
||||
string,
|
||||
{ paths: (string | RegExp)[]; windowMs: number; limit: number }
|
||||
>,
|
||||
|
||||
common: {
|
||||
standardHeaders: "draft-6",
|
||||
keyGenerator: (c: Context) =>
|
||||
(user?.uuid ?? c.req.header("cf-connecting-ip")) +
|
||||
"-" +
|
||||
new Date().getDate(), // day so that limit gets reset every day
|
||||
store: new DurableObjectStore({ namespace: c.env.RATE_LIMITER }),
|
||||
} as GeneralConfigType<ConfigType>,
|
||||
};
|
||||
default: {
|
||||
windowMs: 60 * 1000, // 1 minute
|
||||
limit: 100, // 100 requests per minute
|
||||
},
|
||||
|
||||
common: {
|
||||
standardHeaders: "draft-6",
|
||||
keyGenerator: (c: Context) =>
|
||||
(user?.uuid ?? c.req.header("cf-connecting-ip")) +
|
||||
"-" +
|
||||
new Date().getDate(), // day so that limit gets reset every day
|
||||
store: new DurableObjectStore({ namespace: c.env.RATE_LIMITER }),
|
||||
} as GeneralConfigType<ConfigType>,
|
||||
};
|
||||
|
||||
if (
|
||||
c.req.path &&
|
||||
rateLimitConfig.excludedPaths.some((path) =>
|
||||
typeof path === "string" ? c.req.path === path : path.test(c.req.path)
|
||||
)
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
|
||||
// Check for custom rate limits
|
||||
for (const [_, config] of Object.entries(rateLimitConfig.customLimits)) {
|
||||
if (
|
||||
c.req.path &&
|
||||
rateLimitConfig.excludedPaths.some((path) =>
|
||||
config.paths.some((path) =>
|
||||
typeof path === "string" ? c.req.path === path : path.test(c.req.path)
|
||||
)
|
||||
) {
|
||||
return next();
|
||||
return rateLimiter({
|
||||
windowMs: config.windowMs,
|
||||
limit: config.limit,
|
||||
...rateLimitConfig.common,
|
||||
})(c as any, next);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for custom rate limits
|
||||
for (const [_, config] of Object.entries(rateLimitConfig.customLimits)) {
|
||||
if (
|
||||
config.paths.some((path) =>
|
||||
typeof path === "string" ? c.req.path === path : path.test(c.req.path)
|
||||
)
|
||||
) {
|
||||
return rateLimiter({
|
||||
windowMs: config.windowMs,
|
||||
limit: config.limit,
|
||||
...rateLimitConfig.common,
|
||||
})(c as any, next);
|
||||
}
|
||||
}
|
||||
// Apply default rate limit
|
||||
return rateLimiter({
|
||||
windowMs: rateLimitConfig.default.windowMs,
|
||||
limit: rateLimitConfig.default.limit,
|
||||
...rateLimitConfig.common,
|
||||
})(c as any, next);
|
||||
});
|
||||
|
||||
// Apply default rate limit
|
||||
return rateLimiter({
|
||||
windowMs: rateLimitConfig.default.windowMs,
|
||||
limit: rateLimitConfig.default.limit,
|
||||
...rateLimitConfig.common,
|
||||
})(c as any, next);
|
||||
})
|
||||
.get("/", (c) => {
|
||||
return c.html(<LandingPage />);
|
||||
})
|
||||
// TEMPORARY REDIRECT
|
||||
.all("/api/*", async (c) => {
|
||||
// Get the full URL and path
|
||||
const url = new URL(c.req.url);
|
||||
const path = url.pathname;
|
||||
const newPath = path.replace("/api", "/v1");
|
||||
app.get("/", (c) => {
|
||||
return c.html(<LandingPage />);
|
||||
});
|
||||
|
||||
// Preserve query parameters and build target URL
|
||||
const redirectUrl = "https://api.supermemory.ai" + newPath + url.search;
|
||||
// TEMPORARY REDIRECT
|
||||
app.all("/api/*", async (c) => {
|
||||
// Get the full URL and path
|
||||
const url = new URL(c.req.url);
|
||||
const path = url.pathname;
|
||||
const newPath = path.replace("/api", "/v1");
|
||||
|
||||
// Use c.redirect() for a proper redirect
|
||||
return c.redirect(redirectUrl);
|
||||
})
|
||||
.route("/v1/user", user)
|
||||
.route("/v1/spaces", spacesRoute)
|
||||
.route("/v1", actions)
|
||||
.route("/v1/integrations", integrations)
|
||||
.route("/v1/memories", memories)
|
||||
.get("/v1/session", (c) => {
|
||||
const user = c.get("user");
|
||||
// Preserve query parameters and build target URL
|
||||
const redirectUrl = "https://api.supermemory.ai" + newPath + url.search;
|
||||
|
||||
if (!user) {
|
||||
return c.json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
// Use c.redirect() for a proper redirect
|
||||
return c.redirect(redirectUrl);
|
||||
});
|
||||
|
||||
return c.json({
|
||||
user,
|
||||
app.route("/v1/user", user);
|
||||
app.route("/v1/spaces", spacesRoute);
|
||||
app.route("/v1", actions);
|
||||
app.route("/v1/integrations", integrations);
|
||||
app.route("/v1/memories", memories);
|
||||
|
||||
app.get("/v1/session", (c) => {
|
||||
const user = c.get("user");
|
||||
|
||||
if (!user) {
|
||||
return c.json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
|
||||
return c.json({
|
||||
user,
|
||||
});
|
||||
});
|
||||
|
||||
app.post(
|
||||
"/waitlist",
|
||||
zValidator(
|
||||
"json",
|
||||
z.object({ email: z.string().email(), token: z.string() })
|
||||
),
|
||||
async (c) => {
|
||||
const { email, token } = c.req.valid("json");
|
||||
|
||||
const address = c.req.raw.headers.get("CF-Connecting-IP");
|
||||
|
||||
const idempotencyKey = crypto.randomUUID();
|
||||
const url = "https://challenges.cloudflare.com/turnstile/v0/siteverify";
|
||||
const firstResult = await fetch(url, {
|
||||
body: JSON.stringify({
|
||||
secret: c.env.TURNSTILE_SECRET_KEY,
|
||||
response: token,
|
||||
remoteip: address,
|
||||
idempotency_key: idempotencyKey,
|
||||
}),
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
})
|
||||
.post(
|
||||
"/waitlist",
|
||||
zValidator(
|
||||
"json",
|
||||
z.object({ email: z.string().email(), token: z.string() })
|
||||
),
|
||||
async (c) => {
|
||||
const { email, token } = c.req.valid("json");
|
||||
|
||||
const address = c.req.raw.headers.get("CF-Connecting-IP");
|
||||
const firstOutcome = (await firstResult.json()) as { success: boolean };
|
||||
|
||||
const idempotencyKey = crypto.randomUUID();
|
||||
const url = "https://challenges.cloudflare.com/turnstile/v0/siteverify";
|
||||
const firstResult = await fetch(url, {
|
||||
body: JSON.stringify({
|
||||
secret: c.env.TURNSTILE_SECRET_KEY,
|
||||
response: token,
|
||||
remoteip: address,
|
||||
idempotency_key: idempotencyKey,
|
||||
}),
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!firstOutcome.success) {
|
||||
console.info("Turnstile verification failed", firstOutcome);
|
||||
return c.json(
|
||||
{ error: "Turnstile verification failed" },
|
||||
439 as StatusCode
|
||||
);
|
||||
}
|
||||
|
||||
const firstOutcome = (await firstResult.json()) as { success: boolean };
|
||||
const resend = new Resend(c.env.RESEND_API_KEY);
|
||||
|
||||
if (!firstOutcome.success) {
|
||||
console.info("Turnstile verification failed", firstOutcome);
|
||||
return c.json(
|
||||
{ error: "Turnstile verification failed" },
|
||||
439 as StatusCode
|
||||
);
|
||||
}
|
||||
const db = database(c.env.HYPERDRIVE.connectionString);
|
||||
|
||||
const resend = new Resend(c.env.RESEND_API_KEY);
|
||||
const ip =
|
||||
c.req.header("cf-connecting-ip") ||
|
||||
`${c.req.raw.cf?.asn}-${c.req.raw.cf?.country}-${c.req.raw.cf?.city}-${c.req.raw.cf?.region}-${c.req.raw.cf?.postalCode}`;
|
||||
|
||||
const db = database(c.env.HYPERDRIVE.connectionString);
|
||||
const { success } = await c.env.EMAIL_LIMITER.limit({ key: ip });
|
||||
|
||||
const ip =
|
||||
c.req.header("cf-connecting-ip") ||
|
||||
`${c.req.raw.cf?.asn}-${c.req.raw.cf?.country}-${c.req.raw.cf?.city}-${c.req.raw.cf?.region}-${c.req.raw.cf?.postalCode}`;
|
||||
if (!success) {
|
||||
return c.json({ error: "Rate limit exceeded" }, 429);
|
||||
}
|
||||
|
||||
const { success } = await c.env.EMAIL_LIMITER.limit({ key: ip });
|
||||
|
||||
if (!success) {
|
||||
return c.json({ error: "Rate limit exceeded" }, 429);
|
||||
}
|
||||
|
||||
const message = `Supermemory started as a side project a few months ago when I built it as a hackathon project.
|
||||
const message = `Supermemory started as a side project a few months ago when I built it as a hackathon project.
|
||||
<br></br>
|
||||
you guys loved it too much. like wayy too much. it was embarrassing, because this was not it - it was nothing but a hackathon project.
|
||||
<br></br>
|
||||
|
|
@ -216,26 +230,27 @@ export const app = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
So, it's time to make this good. My vision is to make supermemory the best memory tool on the internet.
|
||||
`;
|
||||
|
||||
try {
|
||||
await db.insert(waitlist).values({ email });
|
||||
await resend.emails.send({
|
||||
from: "Dhravya From Supermemory <waitlist@m.supermemory.com>",
|
||||
to: email,
|
||||
subject: "You're in the waitlist - A personal note from Dhravya",
|
||||
html: `<p>Hi. I'm Dhravya. I'm building Supermemory to help people remember everything.<br></br> ${message} <br></br><br></br>I'll be in touch when we launch! Till then, just reply to this email if you wanna talk :)<br></br>If you want to follow me on X, here's my handle: <a href='https://x.com/dhravyashah'>@dhravyashah</a><br></br><br></br>- Dhravya</p>`,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return c.json({ error: "Failed to add to waitlist" }, 400);
|
||||
}
|
||||
|
||||
return c.json({ success: true });
|
||||
try {
|
||||
await db.insert(waitlist).values({ email });
|
||||
await resend.emails.send({
|
||||
from: "Dhravya From Supermemory <waitlist@m.supermemory.com>",
|
||||
to: email,
|
||||
subject: "You're in the waitlist - A personal note from Dhravya",
|
||||
html: `<p>Hi. I'm Dhravya. I'm building Supermemory to help people remember everything.<br></br> ${message} <br></br><br></br>I'll be in touch when we launch! Till then, just reply to this email if you wanna talk :)<br></br>If you want to follow me on X, here's my handle: <a href='https://x.com/dhravyashah'>@dhravyashah</a><br></br><br></br>- Dhravya</p>`,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return c.json({ error: "Failed to add to waitlist" }, 400);
|
||||
}
|
||||
)
|
||||
.onError((err, c) => {
|
||||
console.error(err);
|
||||
return c.json({ error: "Internal server error" }, 500);
|
||||
});
|
||||
|
||||
return c.json({ success: true });
|
||||
}
|
||||
);
|
||||
|
||||
app.onError((err, c) => {
|
||||
console.error(err);
|
||||
return c.json({ error: "Internal server error" }, 500);
|
||||
});
|
||||
|
||||
export default {
|
||||
fetch: app.fetch,
|
||||
|
|
|
|||
18
apps/backend/src/minimal.ts
Normal file
18
apps/backend/src/minimal.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { fromHono } from "chanfana";
|
||||
import { Hono } from "hono";
|
||||
import { serve as serveHono } from "@hono/node-server";
|
||||
|
||||
const app = fromHono(new Hono());
|
||||
|
||||
app.get("/", (c) => {
|
||||
return c.json({ message: "Hello, world!" });
|
||||
});
|
||||
|
||||
app.get("/entry/:id", (c) => {
|
||||
const id = c.req.param("id");
|
||||
return c.json({ message: `Hello, ${id}!` });
|
||||
});
|
||||
|
||||
serveHono(app, (info) => {
|
||||
console.log(info);
|
||||
});
|
||||
|
|
@ -38,8 +38,11 @@ import {
|
|||
} from "@supermemory/db";
|
||||
import { typeDecider } from "../utils/typeDecider";
|
||||
import { isErr, Ok } from "../errors/results";
|
||||
import { fromHono } from "chanfana";
|
||||
|
||||
const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
|
||||
const actions = fromHono(new Hono<{ Variables: Variables; Bindings: Env }>(), {
|
||||
base: "",
|
||||
})
|
||||
.post(
|
||||
"/chat",
|
||||
zValidator(
|
||||
|
|
@ -1195,3 +1198,4 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
);
|
||||
|
||||
export default actions;
|
||||
import { Context } from 'hono/jsx'
|
||||
|
|
|
|||
|
|
@ -5,173 +5,180 @@ import { getAllNotionPageContents } from "../utils/notion";
|
|||
import { and, eq, or } from "@supermemory/db";
|
||||
import { documents } from "@supermemory/db/schema";
|
||||
import { database } from "@supermemory/db";
|
||||
import { fromHono } from "chanfana";
|
||||
|
||||
const integrations = new Hono<{ Variables: Variables; Bindings: Env }>().get("/notion/import", async (c) => {
|
||||
const user = c.get("user");
|
||||
if (!user) {
|
||||
return c.json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
const integrations = fromHono(
|
||||
new Hono<{ Variables: Variables; Bindings: Env }>(),
|
||||
{
|
||||
base: "",
|
||||
}
|
||||
).get("/notion/import", async (c) => {
|
||||
const user = c.get("user");
|
||||
if (!user) {
|
||||
return c.json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
|
||||
// Create SSE stream
|
||||
const stream = new TransformStream();
|
||||
const writer = stream.writable.getWriter();
|
||||
const encoder = new TextEncoder();
|
||||
// Create SSE stream
|
||||
const stream = new TransformStream();
|
||||
const writer = stream.writable.getWriter();
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
// Create response first so client gets headers immediately
|
||||
const response = new Response(stream.readable, {
|
||||
headers: {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
Connection: "keep-alive",
|
||||
// Required CORS headers for SSE
|
||||
"Access-Control-Allow-Origin": "*",
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
},
|
||||
});
|
||||
// Create response first so client gets headers immediately
|
||||
const response = new Response(stream.readable, {
|
||||
headers: {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
Connection: "keep-alive",
|
||||
// Required CORS headers for SSE
|
||||
"Access-Control-Allow-Origin": "*",
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
},
|
||||
});
|
||||
|
||||
const sendMessage = async (data: Record<string, any>) => {
|
||||
// Proper SSE format requires "data: " prefix and double newline
|
||||
const formattedData = `data: ${JSON.stringify(data)}\n\n`;
|
||||
await writer.write(encoder.encode(formattedData));
|
||||
};
|
||||
const sendMessage = async (data: Record<string, any>) => {
|
||||
// Proper SSE format requires "data: " prefix and double newline
|
||||
const formattedData = `data: ${JSON.stringify(data)}\n\n`;
|
||||
await writer.write(encoder.encode(formattedData));
|
||||
};
|
||||
|
||||
// Start processing in background
|
||||
c.executionCtx.waitUntil(
|
||||
(async () => {
|
||||
try {
|
||||
// Send initial heartbeat
|
||||
await sendMessage({ type: "connected" });
|
||||
// Start processing in background
|
||||
c.executionCtx.waitUntil(
|
||||
(async () => {
|
||||
try {
|
||||
// Send initial heartbeat
|
||||
await sendMessage({ type: "connected" });
|
||||
|
||||
const token = await getDecryptedKV(
|
||||
c.env.ENCRYPTED_TOKENS,
|
||||
`${user.uuid}-notion`,
|
||||
`${c.env.WORKOS_COOKIE_PASSWORD}-${user.uuid}`
|
||||
);
|
||||
const token = await getDecryptedKV(
|
||||
c.env.ENCRYPTED_TOKENS,
|
||||
`${user.uuid}-notion`,
|
||||
`${c.env.WORKOS_COOKIE_PASSWORD}-${user.uuid}`
|
||||
);
|
||||
|
||||
const stringToken = new TextDecoder().decode(token);
|
||||
if (!stringToken) {
|
||||
await sendMessage({ type: "error", error: "No token found" });
|
||||
await writer.close();
|
||||
return;
|
||||
const stringToken = new TextDecoder().decode(token);
|
||||
if (!stringToken) {
|
||||
await sendMessage({ type: "error", error: "No token found" });
|
||||
await writer.close();
|
||||
return;
|
||||
}
|
||||
|
||||
await sendMessage({ type: "progress", progress: 5 });
|
||||
|
||||
// Fetch pages with progress updates
|
||||
const pages = await getAllNotionPageContents(
|
||||
stringToken,
|
||||
async (progress) => {
|
||||
// Map progress from 0-100 to 5-40 range
|
||||
const scaledProgress = Math.floor(5 + (progress * 35) / 100);
|
||||
await sendMessage({ type: "progress", progress: scaledProgress });
|
||||
}
|
||||
);
|
||||
|
||||
await sendMessage({ type: "progress", progress: 5 });
|
||||
await sendMessage({ type: "progress", progress: 40 });
|
||||
|
||||
// Fetch pages with progress updates
|
||||
const pages = await getAllNotionPageContents(
|
||||
stringToken,
|
||||
async (progress) => {
|
||||
// Map progress from 0-100 to 5-40 range
|
||||
const scaledProgress = Math.floor(5 + (progress * 35) / 100);
|
||||
await sendMessage({ type: "progress", progress: scaledProgress });
|
||||
}
|
||||
);
|
||||
let processed = 0;
|
||||
const totalPages = pages.length;
|
||||
|
||||
await sendMessage({ type: "progress", progress: 40 });
|
||||
const db = database(c.env.HYPERDRIVE.connectionString);
|
||||
|
||||
let processed = 0;
|
||||
const totalPages = pages.length;
|
||||
for (const page of pages) {
|
||||
// Calculate document hash for duplicate detection
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(page.content);
|
||||
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const documentHash = hashArray
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
const db = database(c.env.HYPERDRIVE.connectionString);
|
||||
|
||||
for (const page of pages) {
|
||||
// Calculate document hash for duplicate detection
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(page.content);
|
||||
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const documentHash = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
|
||||
|
||||
// Check for duplicates using hash
|
||||
const existingDocs = await db
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(
|
||||
and(
|
||||
eq(documents.userId, user.id),
|
||||
or(
|
||||
eq(documents.contentHash, documentHash),
|
||||
and(
|
||||
eq(documents.type, "notion"),
|
||||
or(
|
||||
eq(documents.url, page.url),
|
||||
eq(documents.raw, page.content)
|
||||
)
|
||||
// Check for duplicates using hash
|
||||
const existingDocs = await db
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(
|
||||
and(
|
||||
eq(documents.userId, user.id),
|
||||
or(
|
||||
eq(documents.contentHash, documentHash),
|
||||
and(
|
||||
eq(documents.type, "notion"),
|
||||
or(
|
||||
eq(documents.url, page.url),
|
||||
eq(documents.raw, page.content)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
)
|
||||
);
|
||||
|
||||
if (existingDocs.length > 0) {
|
||||
await sendMessage({
|
||||
type: "warning",
|
||||
message: `Skipping duplicate page: ${page.title}`
|
||||
});
|
||||
processed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Insert into documents table first
|
||||
try {
|
||||
await db.insert(documents).values({
|
||||
uuid: page.id,
|
||||
userId: user.id,
|
||||
type: "notion",
|
||||
url: page.url,
|
||||
title: page.title,
|
||||
contentHash: documentHash,
|
||||
raw: page.content,
|
||||
});
|
||||
|
||||
await c.env.CONTENT_WORKFLOW.create({
|
||||
params: {
|
||||
userId: user.id,
|
||||
content: page.url,
|
||||
spaces: [],
|
||||
type: "notion",
|
||||
uuid: page.id,
|
||||
url: page.url,
|
||||
prefetched: {
|
||||
contentToVectorize: page.content,
|
||||
contentToSave: page.content,
|
||||
title: page.title,
|
||||
type: "notion",
|
||||
},
|
||||
createdAt: page.createdAt,
|
||||
},
|
||||
id: `${user.id}-${page.id}-${new Date().getTime()}`,
|
||||
});
|
||||
|
||||
processed++;
|
||||
const progress = 50 + Math.floor((processed / totalPages) * 50);
|
||||
await sendMessage({ type: "progress", progress, page: page.title });
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Failed to process page ${page.title}:`, error);
|
||||
await sendMessage({
|
||||
type: "warning",
|
||||
message: `Failed to process page: ${page.title}`,
|
||||
error: error instanceof Error ? error.message : "Unknown error"
|
||||
});
|
||||
processed++;
|
||||
continue;
|
||||
}
|
||||
if (existingDocs.length > 0) {
|
||||
await sendMessage({
|
||||
type: "warning",
|
||||
message: `Skipping duplicate page: ${page.title}`,
|
||||
});
|
||||
processed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await sendMessage({ type: "complete", progress: 100 });
|
||||
await writer.close();
|
||||
} catch (error) {
|
||||
console.error("Import error:", error);
|
||||
await sendMessage({
|
||||
type: "error",
|
||||
error: error instanceof Error ? error.message : "Import failed",
|
||||
});
|
||||
await writer.close();
|
||||
// Insert into documents table first
|
||||
try {
|
||||
await db.insert(documents).values({
|
||||
uuid: page.id,
|
||||
userId: user.id,
|
||||
type: "notion",
|
||||
url: page.url,
|
||||
title: page.title,
|
||||
contentHash: documentHash,
|
||||
raw: page.content,
|
||||
});
|
||||
|
||||
await c.env.CONTENT_WORKFLOW.create({
|
||||
params: {
|
||||
userId: user.id,
|
||||
content: page.url,
|
||||
spaces: [],
|
||||
type: "notion",
|
||||
uuid: page.id,
|
||||
url: page.url,
|
||||
prefetched: {
|
||||
contentToVectorize: page.content,
|
||||
contentToSave: page.content,
|
||||
title: page.title,
|
||||
type: "notion",
|
||||
},
|
||||
createdAt: page.createdAt,
|
||||
},
|
||||
id: `${user.id}-${page.id}-${new Date().getTime()}`,
|
||||
});
|
||||
|
||||
processed++;
|
||||
const progress = 50 + Math.floor((processed / totalPages) * 50);
|
||||
await sendMessage({ type: "progress", progress, page: page.title });
|
||||
} catch (error) {
|
||||
console.error(`Failed to process page ${page.title}:`, error);
|
||||
await sendMessage({
|
||||
type: "warning",
|
||||
message: `Failed to process page: ${page.title}`,
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
});
|
||||
processed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
})()
|
||||
);
|
||||
|
||||
return response;
|
||||
});
|
||||
await sendMessage({ type: "complete", progress: 100 });
|
||||
await writer.close();
|
||||
} catch (error) {
|
||||
console.error("Import error:", error);
|
||||
await sendMessage({
|
||||
type: "error",
|
||||
error: error instanceof Error ? error.message : "Import failed",
|
||||
});
|
||||
await writer.close();
|
||||
}
|
||||
})()
|
||||
);
|
||||
|
||||
export default integrations;
|
||||
return response;
|
||||
});
|
||||
|
||||
export default integrations;
|
||||
|
|
|
|||
|
|
@ -9,8 +9,11 @@ import {
|
|||
contentToSpace,
|
||||
} from "@supermemory/db/schema";
|
||||
import { and, database, desc, eq, or, sql, isNull } from "@supermemory/db";
|
||||
import { fromHono } from "chanfana";
|
||||
|
||||
const memories = new Hono<{ Variables: Variables; Bindings: Env }>()
|
||||
const memories = fromHono(new Hono<{ Variables: Variables; Bindings: Env }>(), {
|
||||
base: "",
|
||||
})
|
||||
.get(
|
||||
"/",
|
||||
zValidator(
|
||||
|
|
|
|||
|
|
@ -12,8 +12,14 @@ import {
|
|||
import { zValidator } from "@hono/zod-validator";
|
||||
import { z } from "zod";
|
||||
import { randomId } from "@supermemory/shared";
|
||||
import { fromHono } from "chanfana";
|
||||
|
||||
const spacesRoute = new Hono<{ Variables: Variables; Bindings: Env }>()
|
||||
const spacesRoute = fromHono(
|
||||
new Hono<{ Variables: Variables; Bindings: Env }>(),
|
||||
{
|
||||
base: "",
|
||||
}
|
||||
)
|
||||
.get("/", async (c) => {
|
||||
const user = c.get("user");
|
||||
if (!user) {
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@ import {
|
|||
import { decryptApiKey, getApiKey } from "../auth";
|
||||
import { DurableObjectStore } from "@hono-rate-limiter/cloudflare";
|
||||
import { rateLimiter } from "hono-rate-limiter";
|
||||
import { fromHono } from "chanfana";
|
||||
|
||||
const user = new Hono<{ Variables: Variables; Bindings: Env }>()
|
||||
const user = fromHono(new Hono<{ Variables: Variables; Bindings: Env }>(), {
|
||||
base: "",
|
||||
})
|
||||
.get("/", (c) => {
|
||||
return c.json(c.get("user"));
|
||||
})
|
||||
|
|
@ -132,30 +135,23 @@ const user = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
|
||||
return c.json({ invitations });
|
||||
})
|
||||
.get(
|
||||
"/key",
|
||||
async (c) => {
|
||||
const user = c.get("user");
|
||||
if (!user) {
|
||||
return c.json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
|
||||
// we need user.id and user.lastApiKeyGeneratedAt
|
||||
const lastApiKeyGeneratedAt = user.lastApiKeyGeneratedAt?.getTime();
|
||||
if (!lastApiKeyGeneratedAt) {
|
||||
return c.json({ error: "No API key generated" }, 400);
|
||||
}
|
||||
|
||||
const key = await getApiKey(
|
||||
user.uuid,
|
||||
lastApiKeyGeneratedAt.toString(),
|
||||
c
|
||||
);
|
||||
|
||||
const decrypted = await decryptApiKey(key, c);
|
||||
return c.json({ key, decrypted });
|
||||
.get("/key", async (c) => {
|
||||
const user = c.get("user");
|
||||
if (!user) {
|
||||
return c.json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
)
|
||||
|
||||
// we need user.id and user.lastApiKeyGeneratedAt
|
||||
const lastApiKeyGeneratedAt = user.lastApiKeyGeneratedAt?.getTime();
|
||||
if (!lastApiKeyGeneratedAt) {
|
||||
return c.json({ error: "No API key generated" }, 400);
|
||||
}
|
||||
|
||||
const key = await getApiKey(user.uuid, lastApiKeyGeneratedAt.toString(), c);
|
||||
|
||||
const decrypted = await decryptApiKey(key, c);
|
||||
return c.json({ key, decrypted });
|
||||
})
|
||||
.post("/update", async (c) => {
|
||||
const user = c.get("user");
|
||||
if (!user) {
|
||||
|
|
@ -194,6 +190,6 @@ const user = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
.where(eq(users.id, user.id));
|
||||
|
||||
return c.json({ success: true });
|
||||
})
|
||||
});
|
||||
|
||||
export default user;
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@
|
|||
"autoevals": "^0.0.106",
|
||||
"aws4fetch": "^1.0.20",
|
||||
"braintrust": "^0.0.171",
|
||||
"chanfana": "^2.6.3",
|
||||
"cheerio": "^1.0.0",
|
||||
"cloudflare": "^3.5.0",
|
||||
"concurrently": "^9.1.0",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue