mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-08-28 05:25:33 +00:00
fix(tools): skip empty memory prompt injection
This commit is contained in:
parent
e651045ac5
commit
1f8ddd7f2c
11 changed files with 444 additions and 20 deletions
|
|
@ -160,10 +160,12 @@ export class SupermemoryInputProcessor implements Processor {
|
|||
queryText || "",
|
||||
)
|
||||
|
||||
const cachedMemories = this.ctx.memoryCache.get(turnKey)
|
||||
if (cachedMemories) {
|
||||
if (this.ctx.memoryCache.has(turnKey)) {
|
||||
const cachedMemories = this.ctx.memoryCache.get(turnKey) ?? ""
|
||||
this.ctx.logger.debug("Using cached memories", { turnKey })
|
||||
messageList.addSystem(cachedMemories, "supermemory")
|
||||
if (cachedMemories) {
|
||||
messageList.addSystem(cachedMemories, "supermemory")
|
||||
}
|
||||
return messageList
|
||||
}
|
||||
|
||||
|
|
@ -183,8 +185,8 @@ export class SupermemoryInputProcessor implements Processor {
|
|||
promptTemplate: this.ctx.promptTemplate,
|
||||
})
|
||||
|
||||
this.ctx.memoryCache.set(turnKey, memories)
|
||||
if (memories) {
|
||||
this.ctx.memoryCache.set(turnKey, memories)
|
||||
messageList.addSystem(memories, "supermemory")
|
||||
this.ctx.logger.debug("Injected memories into system prompt", {
|
||||
length: memories.length,
|
||||
|
|
|
|||
146
packages/tools/src/openai/middleware.empty-memory.test.ts
Normal file
146
packages/tools/src/openai/middleware.empty-memory.test.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import type OpenAI from "openai"
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
|
||||
import { withSupermemory } from "./index"
|
||||
|
||||
const emptyProfile = {
|
||||
profile: { static: [], dynamic: [] },
|
||||
searchResults: { results: [] },
|
||||
}
|
||||
const originalApiKey = process.env.SUPERMEMORY_API_KEY
|
||||
|
||||
function createClient() {
|
||||
const chatCreate = vi.fn(async () => ({}))
|
||||
const responsesCreate = vi.fn(async () => ({}))
|
||||
const client = {
|
||||
chat: { completions: { create: chatCreate } },
|
||||
responses: { create: responsesCreate },
|
||||
} as unknown as OpenAI
|
||||
|
||||
return { client, chatCreate, responsesCreate }
|
||||
}
|
||||
|
||||
function mockProfileResponse(body: unknown) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => body,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.SUPERMEMORY_API_KEY = "test-api-key"
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (originalApiKey === undefined) {
|
||||
delete process.env.SUPERMEMORY_API_KEY
|
||||
} else {
|
||||
process.env.SUPERMEMORY_API_KEY = originalApiKey
|
||||
}
|
||||
vi.restoreAllMocks()
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe("OpenAI middleware memory injection", () => {
|
||||
it.each([
|
||||
"profile",
|
||||
"query",
|
||||
"full",
|
||||
] as const)("leaves chat messages unchanged for empty %s results", async (mode) => {
|
||||
mockProfileResponse(emptyProfile)
|
||||
const { client, chatCreate } = createClient()
|
||||
const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
|
||||
{ role: "user", content: "Hello" },
|
||||
]
|
||||
const wrapped = withSupermemory(client, {
|
||||
containerTag: "user-123",
|
||||
customId: "conversation-123",
|
||||
mode,
|
||||
addMemory: "never",
|
||||
})
|
||||
|
||||
await wrapped.chat.completions.create({ model: "gpt-4o", messages })
|
||||
|
||||
expect(chatCreate).toHaveBeenCalledWith({ model: "gpt-4o", messages })
|
||||
})
|
||||
|
||||
it("does not append whitespace to an existing system message", async () => {
|
||||
mockProfileResponse(emptyProfile)
|
||||
const { client, chatCreate } = createClient()
|
||||
const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
|
||||
{ role: "system", content: "You are helpful." },
|
||||
{ role: "user", content: "Hello" },
|
||||
]
|
||||
const wrapped = withSupermemory(client, {
|
||||
containerTag: "user-123",
|
||||
customId: "conversation-123",
|
||||
mode: "full",
|
||||
addMemory: "never",
|
||||
})
|
||||
|
||||
await wrapped.chat.completions.create({ model: "gpt-4o", messages })
|
||||
|
||||
expect(chatCreate).toHaveBeenCalledWith({ model: "gpt-4o", messages })
|
||||
})
|
||||
|
||||
it.each([
|
||||
"query",
|
||||
"full",
|
||||
] as const)("keeps Responses API instructions unchanged for empty %s results", async (mode) => {
|
||||
mockProfileResponse(emptyProfile)
|
||||
const { client, responsesCreate } = createClient()
|
||||
const wrapped = withSupermemory(client, {
|
||||
containerTag: "user-123",
|
||||
customId: "conversation-123",
|
||||
mode,
|
||||
addMemory: "never",
|
||||
})
|
||||
|
||||
await wrapped.responses.create({
|
||||
model: "gpt-4o",
|
||||
input: "Hello",
|
||||
instructions: "You are helpful.",
|
||||
})
|
||||
|
||||
expect(responsesCreate).toHaveBeenCalledWith({
|
||||
model: "gpt-4o",
|
||||
input: "Hello",
|
||||
instructions: "You are helpful.",
|
||||
})
|
||||
})
|
||||
|
||||
it("still injects non-empty memories", async () => {
|
||||
mockProfileResponse({
|
||||
profile: {
|
||||
static: [{ memory: "User likes TypeScript" }],
|
||||
dynamic: [],
|
||||
},
|
||||
searchResults: { results: [] },
|
||||
})
|
||||
const { client, chatCreate } = createClient()
|
||||
const wrapped = withSupermemory(client, {
|
||||
containerTag: "user-123",
|
||||
customId: "conversation-123",
|
||||
mode: "profile",
|
||||
addMemory: "never",
|
||||
})
|
||||
|
||||
await wrapped.chat.completions.create({
|
||||
model: "gpt-4o",
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
})
|
||||
|
||||
expect(chatCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
messages: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
role: "system",
|
||||
content: expect.stringContaining("User likes TypeScript"),
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
@ -11,6 +11,19 @@ const normalizeBaseUrl = (url?: string): string => {
|
|||
return url.endsWith("/") ? url.slice(0, -1) : url
|
||||
}
|
||||
|
||||
const hasRelevantMemories = (
|
||||
mode: "profile" | "query" | "full",
|
||||
memories: { static: string[]; dynamic: string[]; searchResults: string[] },
|
||||
): boolean => {
|
||||
const hasProfileMemories =
|
||||
mode !== "query" &&
|
||||
(memories.static.length > 0 || memories.dynamic.length > 0)
|
||||
const hasSearchMemories =
|
||||
mode !== "profile" && memories.searchResults.length > 0
|
||||
|
||||
return hasProfileMemories || hasSearchMemories
|
||||
}
|
||||
|
||||
export interface OpenAIMiddlewareOptions {
|
||||
/** Container tag/identifier for memory search (e.g., user ID, project ID). Required. */
|
||||
containerTag: string
|
||||
|
|
@ -31,7 +44,6 @@ interface SupermemoryProfileSearch {
|
|||
results: Array<{ memory: string; metadata?: Record<string, unknown> }>
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the last user message from an array of chat completion messages.
|
||||
*
|
||||
|
|
@ -205,6 +217,11 @@ const addSystemPrompt = async (
|
|||
},
|
||||
})
|
||||
|
||||
if (!hasRelevantMemories(mode, deduplicated)) {
|
||||
logger.debug("No memories found for chat API prompt injection")
|
||||
return messages
|
||||
}
|
||||
|
||||
const profileData =
|
||||
mode !== "query"
|
||||
? convertProfileToMarkdown({
|
||||
|
|
@ -492,6 +509,11 @@ export function createOpenAIMiddleware(
|
|||
},
|
||||
})
|
||||
|
||||
if (!hasRelevantMemories(mode, deduplicated)) {
|
||||
logger.debug(`No memories found for ${context} API prompt injection`)
|
||||
return ""
|
||||
}
|
||||
|
||||
const profileData =
|
||||
mode !== "query"
|
||||
? convertProfileToMarkdown({
|
||||
|
|
|
|||
|
|
@ -23,6 +23,49 @@ afterEach(() => {
|
|||
})
|
||||
|
||||
describe("buildMemoriesText", () => {
|
||||
it.each([
|
||||
"profile",
|
||||
"query",
|
||||
"full",
|
||||
] as const)("returns no prompt when %s mode finds no memories", async (mode) => {
|
||||
mockProfileResponse({
|
||||
profile: { static: [], dynamic: [] },
|
||||
searchResults: { results: [] },
|
||||
})
|
||||
|
||||
const memories = await buildMemoriesText({
|
||||
containerTag: CONTAINER_TAG,
|
||||
queryText: mode === "profile" ? "" : "what do you know about me?",
|
||||
mode,
|
||||
baseUrl: BASE_URL,
|
||||
apiKey: API_KEY,
|
||||
logger,
|
||||
})
|
||||
|
||||
expect(memories).toBe("")
|
||||
})
|
||||
|
||||
it("does not invoke a custom template when no memories are found", async () => {
|
||||
mockProfileResponse({
|
||||
profile: { static: [], dynamic: [] },
|
||||
searchResults: { results: [] },
|
||||
})
|
||||
const promptTemplate = vi.fn(() => "<user_memories></user_memories>")
|
||||
|
||||
const memories = await buildMemoriesText({
|
||||
containerTag: CONTAINER_TAG,
|
||||
queryText: "what do you know about me?",
|
||||
mode: "full",
|
||||
baseUrl: BASE_URL,
|
||||
apiKey: API_KEY,
|
||||
logger,
|
||||
promptTemplate,
|
||||
})
|
||||
|
||||
expect(memories).toBe("")
|
||||
expect(promptTemplate).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// The profile is not injected in "query" mode. Deduplicating the search
|
||||
// results against it would drop a fact present in both, leaving the model
|
||||
// with nothing.
|
||||
|
|
|
|||
|
|
@ -140,6 +140,17 @@ export const buildMemoriesText = async (
|
|||
},
|
||||
})
|
||||
|
||||
const hasProfileMemories =
|
||||
mode !== "query" &&
|
||||
(deduplicated.static.length > 0 || deduplicated.dynamic.length > 0)
|
||||
const hasSearchMemories =
|
||||
mode !== "profile" && deduplicated.searchResults.length > 0
|
||||
|
||||
if (!hasProfileMemories && !hasSearchMemories) {
|
||||
logger.debug("No memories found for prompt injection")
|
||||
return ""
|
||||
}
|
||||
|
||||
const userMemories =
|
||||
mode !== "query"
|
||||
? convertProfileToMarkdown({
|
||||
|
|
|
|||
|
|
@ -61,6 +61,11 @@ export const injectMemoriesIntoParams = (
|
|||
memories: string,
|
||||
logger: Logger,
|
||||
): LanguageModelCallOptions => {
|
||||
if (!memories.trim()) {
|
||||
logger.debug("No memories to inject")
|
||||
return params
|
||||
}
|
||||
|
||||
const systemPromptExists = params.prompt.some(
|
||||
(prompt) => prompt.role === "system",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -330,8 +330,8 @@ export const transformParamsWithMemory = async (
|
|||
const isNewTurn = isNewUserTurn(params)
|
||||
|
||||
// Check if we can use cached memories
|
||||
const cachedMemories = ctx.memoryCache.get(turnKey)
|
||||
if (!isNewTurn && cachedMemories) {
|
||||
if (!isNewTurn && ctx.memoryCache.has(turnKey)) {
|
||||
const cachedMemories = ctx.memoryCache.get(turnKey) ?? ""
|
||||
ctx.logger.debug("Using cached memories: ", {
|
||||
turnKey,
|
||||
})
|
||||
|
|
|
|||
105
packages/tools/src/voltagent/middleware.test.ts
Normal file
105
packages/tools/src/voltagent/middleware.test.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest"
|
||||
import {
|
||||
createSupermemoryContext,
|
||||
enhanceMessagesWithMemories,
|
||||
} from "./middleware"
|
||||
import type { VoltAgentMessage } from "./types"
|
||||
|
||||
const userMessage: VoltAgentMessage = {
|
||||
role: "user",
|
||||
content: "Hello",
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe("enhanceMessagesWithMemories", () => {
|
||||
it("caches an empty profile result without injecting a system message", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
profile: { static: [], dynamic: [] },
|
||||
searchResults: { results: [] },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
const context = createSupermemoryContext("user-123", {
|
||||
apiKey: "test-api-key",
|
||||
customId: "conversation-123",
|
||||
mode: "profile",
|
||||
addMemory: "never",
|
||||
})
|
||||
|
||||
const firstResult = await enhanceMessagesWithMemories(
|
||||
[userMessage],
|
||||
context,
|
||||
)
|
||||
const continuation: VoltAgentMessage[] = [
|
||||
userMessage,
|
||||
{ role: "assistant", content: "Hi there!" },
|
||||
]
|
||||
const secondResult = await enhanceMessagesWithMemories(
|
||||
continuation,
|
||||
context,
|
||||
)
|
||||
|
||||
expect(firstResult).toEqual([userMessage])
|
||||
expect(secondResult).toEqual(continuation)
|
||||
expect(context.memoryCache.size).toBe(1)
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("does not inject a placeholder for an empty advanced search", async () => {
|
||||
const promptTemplate = vi.fn(() => "<memories></memories>")
|
||||
const context = createSupermemoryContext("user-123", {
|
||||
apiKey: "test-api-key",
|
||||
customId: "conversation-123",
|
||||
mode: "query",
|
||||
addMemory: "never",
|
||||
limit: 5,
|
||||
promptTemplate,
|
||||
})
|
||||
const search = vi
|
||||
.spyOn(context.client.search, "memories")
|
||||
.mockResolvedValue({ results: [] } as never)
|
||||
|
||||
const result = await enhanceMessagesWithMemories([userMessage], context)
|
||||
const continuation: VoltAgentMessage[] = [
|
||||
userMessage,
|
||||
{ role: "assistant", content: "Hi there!" },
|
||||
]
|
||||
const continuationResult = await enhanceMessagesWithMemories(
|
||||
continuation,
|
||||
context,
|
||||
)
|
||||
|
||||
expect(result).toEqual([userMessage])
|
||||
expect(continuationResult).toEqual(continuation)
|
||||
expect(search).toHaveBeenCalledTimes(1)
|
||||
expect(promptTemplate).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("still injects non-empty advanced search results", async () => {
|
||||
const context = createSupermemoryContext("user-123", {
|
||||
apiKey: "test-api-key",
|
||||
customId: "conversation-123",
|
||||
mode: "query",
|
||||
addMemory: "never",
|
||||
searchMode: "hybrid",
|
||||
})
|
||||
vi.spyOn(context.client.search, "memories").mockResolvedValue({
|
||||
results: [{ chunk: "A relevant document chunk" }],
|
||||
} as never)
|
||||
|
||||
const result = await enhanceMessagesWithMemories([userMessage], context)
|
||||
|
||||
expect(result[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
role: "system",
|
||||
content: expect.stringContaining("A relevant document chunk"),
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
@ -211,8 +211,8 @@ export const enhanceMessagesWithMemories = async (
|
|||
const turnKey = makeTurnKey(ctx, userMessage || "")
|
||||
const isNewTurn = isNewUserTurn(messages)
|
||||
|
||||
const cachedMemories = ctx.memoryCache.get(turnKey)
|
||||
if (!isNewTurn && cachedMemories) {
|
||||
if (!isNewTurn && ctx.memoryCache.has(turnKey)) {
|
||||
const cachedMemories = ctx.memoryCache.get(turnKey) ?? ""
|
||||
ctx.logger.debug("Using cached memories", { turnKey })
|
||||
return injectMemoriesIntoMessages(
|
||||
messagesToEnhance,
|
||||
|
|
@ -300,21 +300,23 @@ export const enhanceMessagesWithMemories = async (
|
|||
const formattedMemories = response.results
|
||||
.map((result: SearchResult) => {
|
||||
const text = result.memory || result.chunk
|
||||
return text ? `- ${text}` : null
|
||||
return text?.trim() ? `- ${text}` : null
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
|
||||
memories = ctx.promptTemplate
|
||||
? ctx.promptTemplate({
|
||||
userMemories: "",
|
||||
generalSearchMemories: formattedMemories,
|
||||
searchResults: response.results as Array<{
|
||||
memory: string
|
||||
metadata?: Record<string, unknown>
|
||||
}>,
|
||||
})
|
||||
: `The following are relevant memories and context about this user retrieved from previous interactions. Use these to personalize your response:\n\n${formattedMemories}`
|
||||
memories = formattedMemories
|
||||
? ctx.promptTemplate
|
||||
? ctx.promptTemplate({
|
||||
userMemories: "",
|
||||
generalSearchMemories: formattedMemories,
|
||||
searchResults: response.results as Array<{
|
||||
memory: string
|
||||
metadata?: Record<string, unknown>
|
||||
}>,
|
||||
})
|
||||
: `The following are relevant memories and context about this user retrieved from previous interactions. Use these to personalize your response:\n\n${formattedMemories}`
|
||||
: ""
|
||||
} else {
|
||||
memories = await buildMemoriesText({
|
||||
containerTag: ctx.containerTag,
|
||||
|
|
@ -346,6 +348,11 @@ const injectMemoriesIntoMessages = (
|
|||
memories: string,
|
||||
logger: Logger,
|
||||
): VoltAgentMessage[] => {
|
||||
if (!memories.trim()) {
|
||||
logger.debug("No memories to inject")
|
||||
return messages
|
||||
}
|
||||
|
||||
const systemMessageIndex = messages.findIndex((msg) => msg.role === "system")
|
||||
|
||||
if (systemMessageIndex !== -1) {
|
||||
|
|
|
|||
|
|
@ -201,6 +201,44 @@ describe("SupermemoryInputProcessor", () => {
|
|||
expect(systemCall?.args[1]).toBe("supermemory")
|
||||
})
|
||||
|
||||
it("should cache an empty result without injecting a system message", async () => {
|
||||
fetchMock.mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve(createMockProfileResponse()),
|
||||
})
|
||||
|
||||
const processor = new SupermemoryInputProcessor({
|
||||
containerTag: TEST_CONFIG.containerTag,
|
||||
customId: TEST_CONFIG.customId,
|
||||
apiKey: TEST_CONFIG.apiKey,
|
||||
mode: "profile",
|
||||
})
|
||||
const messages: MastraDBMessage[] = [createMessage("user", "Hello")]
|
||||
const firstMessageList = createMockMessageList()
|
||||
const secondMessageList = createMockMessageList()
|
||||
|
||||
await processor.processInput({
|
||||
messages,
|
||||
systemMessages: [],
|
||||
messageList: firstMessageList,
|
||||
abort: vi.fn() as never,
|
||||
retryCount: 0,
|
||||
state: {},
|
||||
})
|
||||
await processor.processInput({
|
||||
messages,
|
||||
systemMessages: [],
|
||||
messageList: secondMessageList,
|
||||
abort: vi.fn() as never,
|
||||
retryCount: 0,
|
||||
state: {},
|
||||
})
|
||||
|
||||
expect(firstMessageList.addSystem).not.toHaveBeenCalled()
|
||||
expect(secondMessageList.addSystem).not.toHaveBeenCalled()
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("should use cached memories on second call with same message", async () => {
|
||||
fetchMock.mockResolvedValue({
|
||||
ok: true,
|
||||
|
|
|
|||
|
|
@ -358,6 +358,51 @@ describe("Unit: withSupermemory", () => {
|
|||
expect(fetchMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("should leave the prompt unchanged when no memories are found", async () => {
|
||||
fetchMock.mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve(createMockProfileResponse()),
|
||||
})
|
||||
|
||||
const ctx = createSupermemoryContext({
|
||||
containerTag: TEST_CONFIG.containerTag,
|
||||
apiKey: TEST_CONFIG.apiKey,
|
||||
customId: "test-id",
|
||||
mode: "profile",
|
||||
})
|
||||
|
||||
const params: LanguageModelV2CallOptions = {
|
||||
prompt: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Hello" }],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const result = await transformParamsWithMemory(params, ctx)
|
||||
|
||||
expect(result).toEqual(params)
|
||||
expect(ctx.memoryCache.size).toBe(1)
|
||||
|
||||
const continuationParams: LanguageModelV2CallOptions = {
|
||||
prompt: [
|
||||
...params.prompt,
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Hi there!" }],
|
||||
},
|
||||
],
|
||||
}
|
||||
const continuationResult = await transformParamsWithMemory(
|
||||
continuationParams,
|
||||
ctx,
|
||||
)
|
||||
|
||||
expect(continuationResult).toEqual(continuationParams)
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("should handle user message with empty content array in query mode", async () => {
|
||||
const ctx = createSupermemoryContext({
|
||||
containerTag: TEST_CONFIG.containerTag,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue