diff --git a/packages/tools/package.json b/packages/tools/package.json index 081e50d1..8e9fb55c 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -1,7 +1,7 @@ { "name": "@supermemory/tools", "type": "module", - "version": "3.1.0", + "version": "3.1.1", "description": "Memory tools for AI SDK, OpenAI, Voltagent and Mastra with supermemory", "scripts": { "build": "tsdown", @@ -39,6 +39,9 @@ "ai": "^5.0.0 || ^6.0.0 || ^7.0.0" }, "peerDependenciesMeta": { + "@ai-sdk/provider": { + "optional": true + }, "@voltagent/core": { "optional": true }, diff --git a/packages/tools/src/vercel/middleware.ts b/packages/tools/src/vercel/middleware.ts index d7e024b1..922e2d16 100644 --- a/packages/tools/src/vercel/middleware.ts +++ b/packages/tools/src/vercel/middleware.ts @@ -3,7 +3,6 @@ import { addConversation, type ContentPart, type ConversationMessage, - toConversationImageUrl, } from "../conversations-client" import { createLogger, @@ -14,7 +13,11 @@ import { type PromptTemplate, type MemoryMode, } from "../shared" -import { type LanguageModelCallOptions, getLastUserMessage } from "./util" +import { + type LanguageModelCallOptions, + getLastUserMessage, + imageFilePartUrl, +} from "./util" import { extractQueryText, injectMemoriesIntoParams } from "./memory-prompt" const safeJsonStringify = (value: unknown): string => { @@ -104,11 +107,8 @@ export const convertToConversationMessages = ( type: "text", text: content.text, }) - } else if ( - content.type === "file" && - content.mediaType.startsWith("image/") - ) { - const url = toConversationImageUrl(content.data, content.mediaType) + } else if (content.type === "file") { + const url = imageFilePartUrl(content) if (url) { contentParts.push({ type: "image_url", imageUrl: { url } }) } diff --git a/packages/tools/src/vercel/util.ts b/packages/tools/src/vercel/util.ts index 640eee11..413f75f2 100644 --- a/packages/tools/src/vercel/util.ts +++ b/packages/tools/src/vercel/util.ts @@ -75,6 +75,31 @@ export const getLastUserMessage = ( .join(" ") } +// AI SDK 7 tags file data ({ type: "data" | "url" }) and allows a bare "image" media type. +export const imageFilePartUrl = (part: { + mediaType?: unknown + data?: unknown +}): string | null => { + const { mediaType, data } = part + if (typeof mediaType !== "string") return null + if (mediaType !== "image" && !mediaType.startsWith("image/")) return null + const tagged = + data && typeof data === "object" && "type" in data + ? (data as { type: unknown; data?: unknown; url?: unknown }) + : null + const value = !tagged + ? data + : tagged.type === "data" + ? tagged.data + : tagged.type === "url" + ? tagged.url + : null + return toConversationImageUrl( + value, + mediaType === "image" ? undefined : mediaType, + ) +} + /** Whether the prompt contains user content that a saved conversation can store. */ export const hasPersistableUserContent = ( params: LanguageModelCallOptions, @@ -97,12 +122,7 @@ export const hasPersistableUserContent = ( if (part.type === "text" && typeof part.text === "string") { return Boolean(part.text.trim()) } - return ( - part.type === "file" && - typeof part.mediaType === "string" && - part.mediaType.startsWith("image/") && - toConversationImageUrl(part.data, part.mediaType) !== null - ) + return part.type === "file" && imageFilePartUrl(part) !== null }) }) } diff --git a/packages/tools/test/ai-sdk-compat/run.ts b/packages/tools/test/ai-sdk-compat/run.ts index b46108a0..23cc5d8d 100644 --- a/packages/tools/test/ai-sdk-compat/run.ts +++ b/packages/tools/test/ai-sdk-compat/run.ts @@ -42,10 +42,12 @@ import { ${v.mock} } from "ai/test" import { supermemoryTools, withSupermemory } from "@supermemory/tools/ai-sdk" let searchCalls = 0 +const saved: string[] = [] const server = Bun.serve({ port: 0, - fetch(req) { + async fetch(req) { const route = new URL(req.url).pathname + if (route.endsWith("/document")) saved.push(String((await req.json()).content)) if (route.endsWith("/profile")) return Response.json({ profile: { static: [{ id: "s0", memory: "User prefers dark mode" }], dynamic: [], buckets: {} } }) if (route.endsWith("/search")) { @@ -60,6 +62,7 @@ const baseUrl = \`http://127.0.0.1:\${server.port}\` const prompts: string[] = [] let calls = 0 const model = new ${v.mock}({ + supportedUrls: { "image/*": [/^https:/] }, doGenerate: async (options) => { prompts.push(JSON.stringify(options.prompt)) const callTool = calls++ === 0 && (options.tools?.length ?? 0) > 0 @@ -83,6 +86,17 @@ const withMemory = await generateText({ model: wrapped, prompt: "hi" }) if (withMemory.text !== "ok" || !prompts.at(-1)?.includes("User prefers dark mode")) throw new Error("withSupermemory: memory was not injected into the prompt") +// Saving is fire-and-forget, so wait for the document request to land. +const savedAfter = async (count: number) => { + for (let i = 0; i < 50 && saved.length < count; i++) await Bun.sleep(20) + return saved[count - 1] ?? "" +} +await savedAfter(1) +await generateText({ model: wrapped, messages: [{ role: "user", content: [{ type: "image", image: new URL("https://example.com/cat.png") }] }] }) +if (!(await savedAfter(saved.length + 1)).includes("[image")) throw new Error("withSupermemory: image-only turn was not saved") +await generateText({ model: wrapped, messages: [{ role: "user", content: [{ type: "text", text: "look" }, { type: "image", image: new Uint8Array([137, 80, 78, 71]), mediaType: "image/png" }] }] }) +if (!(await savedAfter(saved.length + 1)).includes("[image")) throw new Error("withSupermemory: image was dropped from a mixed turn") + // Type-only: the wrapped model and tools must also be accepted by streamText. const _typecheckStream = () => streamText({ model: wrapped, tools, prompt: "hi" }) @@ -143,6 +157,18 @@ try { throw new Error( `ai@${v.ai}: a second copy of ai was installed under @supermemory/tools`, ) + const aiProvider = ( + await Bun.file(path.join(app, "node_modules/ai/package.json")).json() + ).dependencies["@ai-sdk/provider"] + const topProvider = ( + await Bun.file( + path.join(app, "node_modules/@ai-sdk/provider/package.json"), + ).json() + ).version + if (aiProvider.split(".")[0] !== topProvider.split(".")[0]) + throw new Error( + `ai@${installed}: @ai-sdk/provider@${topProvider} was installed, but ai uses ${aiProvider}`, + ) const tsc = await $`npx tsc -p .`.cwd(app).nothrow().quiet() if (tsc.exitCode !== 0) throw new Error(`ai@${installed}: typecheck failed\n${tsc.stdout}`)