From bcfd64c73803a896b2636ae4305edb02aed882e0 Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Fri, 14 Aug 2026 22:58:55 +0530 Subject: [PATCH] fix(tools): handle structured Responses input and keep conversation ids stable Addresses review feedback on the Responses API memory routing: - The wrapper collapsed any non-string `input` to "" (`typeof input === "string" ? input : ""`), so structured or multi-modal Responses requests skipped both memory search and saving. Add extractResponsesInput to flatten string or structured array input into a text query plus role-tagged messages, and route the reconstructed messages through /v4/conversations. - Stop rewriting the stored customId in the client.add fallback. The backend already stores conversation memories under the "conversation:" id, so stripping the prefix would orphan a conversation's earlier turns. Unit tests cover string, empty, structured multi-part, multi-turn, role normalization, and text-less items. --- packages/tools/src/openai/middleware.ts | 108 +++++++++++++++--- .../with-supermemory/responses-input.test.ts | 90 +++++++++++++++ 2 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 packages/tools/test/with-supermemory/responses-input.test.ts diff --git a/packages/tools/src/openai/middleware.ts b/packages/tools/src/openai/middleware.ts index 78df52da..36b1fe48 100644 --- a/packages/tools/src/openai/middleware.ts +++ b/packages/tools/src/openai/middleware.ts @@ -280,6 +280,85 @@ const getConversationContent = ( .join("\n\n") } +/** + * Flattens the Responses API `input` into a text query and chat-style messages. + * + * `input` may be a plain string or an array of structured items where each + * message carries multi-part content (`input_text`, `output_text`, images). + * The previous `typeof input === "string" ? input : ""` collapsed every + * structured or multi-modal request to an empty string, so those turns skipped + * both memory search and saving. Here we pull out the text for search and + * rebuild role-tagged messages so the save still routes through + * /v4/conversations. + * + * @param input - The Responses API `input` value (string or structured array) + * @returns The joined text query and the reconstructed messages + */ +export const extractResponsesInput = ( + input: unknown, +): { + text: string + messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] +} => { + if (typeof input === "string") { + return input.trim() + ? { text: input, messages: [{ role: "user", content: input }] } + : { text: "", messages: [] } + } + + if (!Array.isArray(input)) { + return { text: "", messages: [] } + } + + const textParts: string[] = [] + const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [] + + const pushMessage = (role: string, text: string) => { + const normalizedRole = + role === "assistant" || role === "system" ? role : "user" + messages.push({ + role: normalizedRole, + content: text, + } as OpenAI.Chat.Completions.ChatCompletionMessageParam) + } + + const textFromContentPart = (part: unknown): string => { + if (typeof part === "string") return part + if (!part || typeof part !== "object") return "" + const record = part as Record + return typeof record.text === "string" ? record.text : "" + } + + for (const item of input) { + if (typeof item === "string") { + if (item.trim()) { + textParts.push(item) + pushMessage("user", item) + } + continue + } + if (!item || typeof item !== "object") continue + + const record = item as Record + const role = typeof record.role === "string" ? record.role : "user" + const content = record.content + + const text = + typeof content === "string" + ? content + : Array.isArray(content) + ? content.map(textFromContentPart).filter(Boolean).join("\n") + : "" + + if (text.trim()) { + textParts.push(text) + pushMessage(role, text) + } + } + + return { text: textParts.join("\n").trim(), messages } +} + /** * Adds a new memory to the SuperMemory system. * @@ -367,13 +446,14 @@ const addMemoryTool = async ( return } - // Fallback to old behavior for non-conversation memories. The - // "conversation:" prefix is an internal routing marker — never let - // it reach the stored customId (colons fail customId validation). + // Fallback for when we cannot reach /v4/conversations (no messages or + // API key). Keep the customId exactly as passed: the backend already + // stores conversation memories under the "conversation:" id, so + // rewriting it here would orphan a conversation's earlier turns. const response = await client.add({ content, containerTags: [containerTag], - customId: customId?.replace(/^conversation:/, ""), + customId, }) logger.info("Memory saved successfully", { @@ -532,7 +612,9 @@ export function createOpenAIMiddleware( ) } - const input = typeof params.input === "string" ? params.input : "" + const { text: input, messages: inputMessages } = extractResponsesInput( + params.input, + ) if (mode !== "profile" && !input) { logger.debug("No input found for Responses API, skipping memory search") @@ -550,15 +632,11 @@ export function createOpenAIMiddleware( if (addMemory === "always" && input?.trim()) { const content = customId ? `Input: ${input}` : input const memoryCustomId = customId ? `conversation:${customId}` : undefined - // Mirror the chat path: hand the input over as messages together - // with the credentials so a configured customId routes through - // /v4/conversations. Without these, addMemoryTool fell through to - // client.add() with the internal "conversation:" prefix baked - // into the stored customId — which the API rejects, so Responses - // API memory saves silently failed whenever customId was set. - const inputAsMessages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = - [{ role: "user", content: input }] - + // Mirror the chat path: hand the reconstructed messages over with + // the credentials so a configured customId routes through + // /v4/conversations. Without messages, addMemoryTool falls back to + // client.add() with the "conversation:" customId; passing them lets + // structured and multi-turn Responses input be saved intact. operations.push( addMemoryTool( client, @@ -566,7 +644,7 @@ export function createOpenAIMiddleware( content, memoryCustomId, logger, - customId ? inputAsMessages : undefined, + customId ? inputMessages : undefined, process.env.SUPERMEMORY_API_KEY, baseUrl, ), diff --git a/packages/tools/test/with-supermemory/responses-input.test.ts b/packages/tools/test/with-supermemory/responses-input.test.ts new file mode 100644 index 00000000..84d95b66 --- /dev/null +++ b/packages/tools/test/with-supermemory/responses-input.test.ts @@ -0,0 +1,90 @@ +/** + * Unit tests for extractResponsesInput. + * + * The Responses API `input` can be a plain string or a structured array of + * items with multi-part content. The wrapper previously collapsed anything + * that was not a string to "", which silently skipped memory search and + * saving for structured or multi-modal requests. + */ + +import { describe, it, expect } from "vitest" +import { extractResponsesInput } from "../../src/openai/middleware" + +describe("extractResponsesInput", () => { + it("passes a plain string through as a single user message", () => { + const result = extractResponsesInput("Where do I live?") + + expect(result.text).toBe("Where do I live?") + expect(result.messages).toEqual([ + { role: "user", content: "Where do I live?" }, + ]) + }) + + it("returns nothing for an empty string", () => { + const result = extractResponsesInput(" ") + + expect(result.text).toBe("") + expect(result.messages).toEqual([]) + }) + + it("returns nothing for unsupported input types", () => { + expect(extractResponsesInput(undefined)).toEqual({ text: "", messages: [] }) + expect(extractResponsesInput(null)).toEqual({ text: "", messages: [] }) + expect(extractResponsesInput(42)).toEqual({ text: "", messages: [] }) + }) + + it("extracts text from structured multi-part content", () => { + const result = extractResponsesInput([ + { + role: "user", + content: [ + { type: "input_text", text: "Describe this" }, + { type: "input_image", image_url: "https://example.com/cat.png" }, + { type: "input_text", text: "in one word" }, + ], + }, + ]) + + expect(result.text).toBe("Describe this\nin one word") + expect(result.messages).toEqual([ + { role: "user", content: "Describe this\nin one word" }, + ]) + }) + + it("preserves roles across a multi-turn structured input", () => { + const result = extractResponsesInput([ + { role: "user", content: "Hi" }, + { role: "assistant", content: "Hello!" }, + { role: "user", content: [{ type: "input_text", text: "How are you?" }] }, + ]) + + expect(result.text).toBe("Hi\nHello!\nHow are you?") + expect(result.messages).toEqual([ + { role: "user", content: "Hi" }, + { role: "assistant", content: "Hello!" }, + { role: "user", content: "How are you?" }, + ]) + }) + + it("normalizes unknown or developer roles to user", () => { + const result = extractResponsesInput([ + { role: "developer", content: "system-ish note" }, + ]) + + expect(result.messages).toEqual([ + { role: "user", content: "system-ish note" }, + ]) + }) + + it("skips items that carry no text", () => { + const result = extractResponsesInput([ + { role: "user", content: [{ type: "input_image", image_url: "x" }] }, + { role: "user", content: "actual question" }, + ]) + + expect(result.text).toBe("actual question") + expect(result.messages).toEqual([ + { role: "user", content: "actual question" }, + ]) + }) +})