feat(python-sdks): SDK-level cross-source memory deduplication

Port the normalized, priority-ordered (static > dynamic > search) profile
deduplication into the Python SDKs, injecting one owned memory block per
request that replaces the prior block rather than accumulating. Dedup is
request-local (no shared state), so it stays correct under concurrency.

Covers OpenAI, Agent Framework (middleware + context provider), Cartesia,
and Pipecat.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

packages/agent-framework-python/src/supermemory_agent_framework/context_provider.py

@@ -217,8 +217,8 @@ class SupermemoryContextProvider(BaseContextProvider):
         )
 
         deduplicated = deduplicate_memories(
-            static=static,
-            dynamic=dynamic,
+            static=static if self._mode != "query" else [],
+            dynamic=dynamic if self._mode != "query" else [],
             search_results=search_results_raw,
         )
 

packages/agent-framework-python/src/supermemory_agent_framework/middleware.py

@@ -21,6 +21,8 @@ from .utils import (
     convert_profile_to_markdown,
     create_logger,
     deduplicate_memories,
+    replace_memory_injection,
+    strip_memory_injection,
     wrap_memory_injection,
 )
 
@@ -152,8 +154,8 @@ async def _build_memories_text(
     )
 
     deduplicated = deduplicate_memories(
-        static=static,
-        dynamic=dynamic,
+        static=static if mode != "query" else [],
+        dynamic=dynamic if mode != "query" else [],
         search_results=search_results_raw,
     )
 
@@ -393,10 +395,11 @@ def _inject_memories(context: Any, memories: str) -> None:
     different Agent Framework providers.
     """
     messages = context.messages
-    memory_text = f"\n\n{wrap_memory_injection(memories)}"
+    memory_text = wrap_memory_injection(memories)
 
-    # Try to find and augment existing system message
-    for i, msg in enumerate(messages):
+    # Replace prior SDK blocks in every system message and inject once.
+    injected = False
+    for msg in messages:
         role = None
         if hasattr(msg, "role"):
             role = msg.role
@@ -405,17 +408,35 @@ def _inject_memories(context: Any, memories: str) -> None:
 
         if role == "system":
             if hasattr(msg, "text"):
-                msg.text = (msg.text or "") + memory_text
+                existing = msg.text or ""
+                msg.text = (
+                    replace_memory_injection(existing, memories)
+                    if not injected
+                    else strip_memory_injection(existing)
+                )
             elif hasattr(msg, "content"):
-                msg.content = (msg.content or "") + memory_text
+                existing = msg.content or ""
+                msg.content = (
+                    replace_memory_injection(existing, memories)
+                    if not injected
+                    else strip_memory_injection(existing)
+                )
             elif isinstance(msg, dict):
-                msg["content"] = (msg.get("content", "") or "") + memory_text
-            return
+                existing = msg.get("content", "") or ""
+                msg["content"] = (
+                    replace_memory_injection(existing, memories)
+                    if not injected
+                    else strip_memory_injection(existing)
+                )
+            injected = True
+
+    if injected:
+        return
 
     # No system message found - prepend one
     try:
         if isinstance(messages, list):
-            messages.insert(0, Message("system", [memories]))
+            messages.insert(0, Message("system", [memory_text]))
     except Exception:
         # If messages is immutable, log a warning
         pass

packages/agent-framework-python/src/supermemory_agent_framework/utils.py

@@ -5,6 +5,10 @@ import re
 from typing import Any, Optional, Protocol
 
 DEFAULT_CONTEXT_PROMPT = "The following are retrieved memories about the user."
+MEMORY_CONTEXT_PATTERN = re.compile(
+    r'[ \t]*<supermemory context="user-memories" readonly>.*?</supermemory>[ \t]*',
+    re.DOTALL,
+)
 
 
 def wrap_memory_injection(memories: str, context_prompt: str = "") -> str:
@@ -19,6 +23,21 @@ def wrap_memory_injection(memories: str, context_prompt: str = "") -> str:
     )
 
 
+def strip_memory_injection(content: str) -> str:
+    """Remove every context block previously owned by this middleware."""
+    stripped = MEMORY_CONTEXT_PATTERN.sub("", content)
+    return re.sub(r"\n{3,}", "\n\n", stripped).strip()
+
+
+def replace_memory_injection(content: str, memories: str) -> str:
+    """Replace middleware-owned context while preserving caller instructions."""
+    preserved = strip_memory_injection(content)
+    memory_context = wrap_memory_injection(memories) if memories.strip() else ""
+    if not memory_context:
+        return preserved
+    return f"{preserved}\n\n{memory_context}" if preserved else memory_context
+
+
 class Logger(Protocol):
     """Logger protocol for type safety."""
 
@@ -110,36 +129,40 @@ def deduplicate_memories(
         return None
 
     def comparison_key(memory: str) -> str:
-        """Remove Mono's dynamic-profile date decoration for comparison only."""
-        return re.sub(
+        """Normalize display-only profile decoration for duplicate comparison."""
+        without_prefix = re.sub(
             r"^(?:\[Recent\]\s*)?\[\d{4}-\d{2}-\d{2}\]\s*",
             "",
             memory,
             count=1,
-        ).strip()
+        )
+        return " ".join(without_prefix.strip().split()).casefold()
 
     static_memories: list[str] = []
     seen_memories: set[str] = set()
 
     for item in static_items:
         memory = extract_memory_text(item)
-        if memory is not None:
+        key = comparison_key(memory) if memory is not None else None
+        if memory is not None and key is not None and key not in seen_memories:
             static_memories.append(memory)
-            seen_memories.add(comparison_key(memory))
+            seen_memories.add(key)
 
     dynamic_memories: list[str] = []
     for item in dynamic_items:
         memory = extract_memory_text(item)
-        if memory is not None and comparison_key(memory) not in seen_memories:
+        key = comparison_key(memory) if memory is not None else None
+        if memory is not None and key is not None and key not in seen_memories:
             dynamic_memories.append(memory)
-            seen_memories.add(comparison_key(memory))
+            seen_memories.add(key)
 
     search_memories: list[str] = []
     for item in search_items:
         memory = extract_memory_text(item)
-        if memory is not None and comparison_key(memory) not in seen_memories:
+        key = comparison_key(memory) if memory is not None else None
+        if memory is not None and key is not None and key not in seen_memories:
             search_memories.append(memory)
-            seen_memories.add(comparison_key(memory))
+            seen_memories.add(key)
 
     return DeduplicatedMemories(
         static=static_memories,

packages/agent-framework-python/tests/test_context_provider.py

@@ -1,5 +1,8 @@
 """Tests for Supermemory context provider."""
 
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
 import pytest
 
 from supermemory_agent_framework import AgentSupermemory, SupermemoryContextProvider
@@ -123,3 +126,23 @@ class TestExtractConversation:
         result = provider._extract_conversation_from_context(MockContext())
         assert "User: Hello!" in result
         assert "Assistant: Hi there!" in result
+
+
+class TestMemoryRetrieval:
+    @pytest.mark.asyncio
+    async def test_query_mode_keeps_search_fact_also_present_in_profile(self) -> None:
+        fact = "User likes machine learning projects"
+        conn = _make_conn()
+        conn.client.profile = AsyncMock(
+            return_value=SimpleNamespace(
+                profile=SimpleNamespace(static=[fact], dynamic=[]),
+                search_results=SimpleNamespace(
+                    results=[SimpleNamespace(memory=fact)]
+                ),
+            )
+        )
+        provider = SupermemoryContextProvider(conn, mode="query")
+
+        memories = await provider._fetch_memories("machine learning")
+
+        assert fact in memories

packages/agent-framework-python/tests/test_middleware.py

@@ -1,5 +1,8 @@
 """Tests for Supermemory middleware."""
 
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
 import pytest
 
 from supermemory_agent_framework import (
@@ -10,6 +13,8 @@ from supermemory_agent_framework import (
 from supermemory_agent_framework.middleware import (
     _get_last_user_message,
     _get_conversation_content,
+    _build_memories_text,
+    _inject_memories,
 )
 
 
@@ -111,3 +116,52 @@ class TestMiddlewareConfiguration:
         conn = _make_conn(entity_context="User is a Python developer")
         middleware = SupermemoryChatMiddleware(conn)
         assert middleware._connection.entity_context == "User is a Python developer"
+
+
+class TestMemoryInjection:
+    def test_replaces_prior_sdk_context(self) -> None:
+        context = SimpleNamespace(
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "Be helpful.\n\n"
+                        '<supermemory context="user-memories" readonly>\n'
+                        "Stale profile fact\n"
+                        "</supermemory>"
+                    ),
+                },
+                {"role": "user", "content": "What do you remember?"},
+            ]
+        )
+
+        _inject_memories(context, "Fresh profile fact")
+
+        content = context.messages[0]["content"]
+        assert "Be helpful." in content
+        assert "Fresh profile fact" in content
+        assert "Stale profile fact" not in content
+        assert content.count(
+            '<supermemory context="user-memories" readonly>'
+        ) == 1
+
+    @pytest.mark.asyncio
+    async def test_query_mode_keeps_search_fact_also_present_in_profile(self) -> None:
+        fact = "User likes machine learning projects"
+        client = SimpleNamespace(
+            profile=AsyncMock(
+                return_value=SimpleNamespace(
+                    profile=SimpleNamespace(static=[fact], dynamic=[]),
+                    search_results=SimpleNamespace(
+                        results=[SimpleNamespace(memory=fact)]
+                    ),
+                )
+            )
+        )
+        logger = Mock()
+
+        memories = await _build_memories_text(
+            "user-123", logger, "query", client, "machine learning"
+        )
+
+        assert fact in memories

packages/agent-framework-python/tests/test_utils.py

@@ -56,6 +56,14 @@ class TestDeduplicateMemories:
         )
         assert result.static == ["valid"]
 
+    def test_normalized_fact_variants(self) -> None:
+        result = deduplicate_memories(
+            static=["User likes Python", " user likes python "],
+            dynamic=["[2026-08-10] USER LIKES PYTHON"],
+        )
+        assert result.static == ["User likes Python"]
+        assert result.dynamic == []
+
 
 class TestConvertProfileToMarkdown:
     def test_empty_profile(self) -> None:

packages/cartesia-sdk-python/src/supermemory_cartesia/agent.py

@@ -237,9 +237,11 @@ class SupermemoryCartesiaAgent:
     def _build_memory_message(self, memories_data: Dict[str, Any]) -> Optional[str]:
         """Build memory context from retrieved data."""
         profile = memories_data["profile"]
+        include_profile = self.config.mode in ("profile", "full")
+        include_search = self.config.mode in ("query", "full")
         deduplicated = deduplicate_memories(
-            static=profile["static"],
-            dynamic=profile["dynamic"],
+            static=profile["static"] if include_profile else [],
+            dynamic=profile["dynamic"] if include_profile else [],
             search_results=memories_data["search_results"],
         )
 
@@ -252,9 +254,6 @@ class SupermemoryCartesiaAgent:
         if total == 0:
             return None
 
-        include_profile = self.config.mode in ("profile", "full")
-        include_search = self.config.mode in ("query", "full")
-
         memory_text = format_memories_to_text(
             deduplicated,
             system_prompt=self.config.system_prompt,

packages/cartesia-sdk-python/tests/test_empty_profile.py

@@ -72,6 +72,26 @@ class TestSupermemoryCartesiaNullProfile(unittest.IsolatedAsyncioTestCase):
             },
         )
 
+    def test_query_mode_keeps_search_fact_also_present_in_profile(self) -> None:
+        fact = "User likes machine learning projects"
+        agent = SupermemoryCartesiaAgent(
+            agent=SimpleNamespace(),
+            api_key="mock_key",
+            container_tag="user-123",
+            custom_id="conversation-456",
+            config=SupermemoryCartesiaAgent.MemoryConfig(mode="query"),
+        )
+
+        context = agent._build_memory_message(
+            {
+                "profile": {"static": [fact], "dynamic": []},
+                "search_results": [SimpleNamespace(memory=fact)],
+            }
+        )
+
+        self.assertIsNotNone(context)
+        self.assertIn(fact, context)
+
 
 if __name__ == "__main__":
     unittest.main()

packages/openai-sdk-python/src/supermemory_openai/middleware.py

@@ -26,6 +26,9 @@ from .utils import (
     deduplicate_memories,
     get_conversation_content,
     get_last_user_message,
+    replace_memory_context,
+    strip_memory_context,
+    wrap_memory_context,
 )
 
 DEFAULT_SUPERMEMORY_BASE_URL = "https://api.supermemory.ai"
@@ -155,8 +158,8 @@ async def add_system_prompt(
     )
 
     deduplicated = deduplicate_memories(
-        static=profile.get("static", []),
-        dynamic=profile.get("dynamic", []),
+        static=profile.get("static", []) if mode != "query" else [],
+        dynamic=profile.get("dynamic", []) if mode != "query" else [],
         search_results=search_results_data.get("results", []),
     )
 
@@ -208,24 +211,40 @@ async def add_system_prompt(
             },
         )
 
+    if system_prompt_exists:
+        logger.debug("Replaced Supermemory context in existing system prompt")
+        enhanced: list[ChatCompletionMessageParam] = []
+        injected = False
+        for msg in messages:
+            if msg.get("role") != "system":
+                enhanced.append(msg)
+                continue
+            content = msg.get("content", "")
+            existing = content if isinstance(content, str) else ""
+            if not injected:
+                enhanced.append(
+                    cast(
+                        ChatCompletionMessageParam,
+                        {**msg, "content": replace_memory_context(existing, memories)},
+                    )
+                )
+                injected = True
+            else:
+                enhanced.append(
+                    cast(
+                        ChatCompletionMessageParam,
+                        {**msg, "content": strip_memory_context(existing)},
+                    )
+                )
+        return enhanced
+
     if not memories:
         return messages
 
-    if system_prompt_exists:
-        logger.debug("Added memories to existing system prompt")
-        return [
-            (
-                {**msg, "content": f"{msg.get('content', '')} \n {memories}"}
-                if msg.get("role") == "system"
-                else msg
-            )
-            for msg in messages
-        ]
-
     logger.debug("System prompt does not exist, created system prompt with memories")
     system_message: ChatCompletionSystemMessageParam = {
         "role": "system",
-        "content": memories,
+        "content": wrap_memory_context(memories),
     }
     return [system_message] + messages
 

packages/openai-sdk-python/src/supermemory_openai/utils.py

@@ -1,11 +1,43 @@
 """Utility functions for Supermemory OpenAI middleware."""
 
 import json
+import re
 from typing import Optional, Any, Protocol
 
 from openai.types.chat import ChatCompletionMessageParam
 
 
+MEMORY_CONTEXT_START = '<supermemory context="user-memories" readonly>'
+MEMORY_CONTEXT_END = "</supermemory>"
+MEMORY_CONTEXT_PATTERN = re.compile(
+    r'[ \t]*<supermemory context="user-memories" readonly>.*?</supermemory>[ \t]*',
+    re.DOTALL,
+)
+
+
+def strip_memory_context(content: str) -> str:
+    """Remove every context block previously owned by this middleware."""
+    stripped = MEMORY_CONTEXT_PATTERN.sub("", content)
+    return re.sub(r"\n{3,}", "\n\n", stripped).strip()
+
+
+def wrap_memory_context(memories: str) -> str:
+    """Mark retrieved context so the next turn can replace it safely."""
+    normalized = memories.strip()
+    if not normalized:
+        return ""
+    return f"{MEMORY_CONTEXT_START}\n{normalized}\n{MEMORY_CONTEXT_END}"
+
+
+def replace_memory_context(content: str, memories: str) -> str:
+    """Replace middleware-owned context while preserving caller instructions."""
+    preserved = strip_memory_context(content)
+    memory_context = wrap_memory_context(memories)
+    if not memory_context:
+        return preserved
+    return f"{preserved}\n\n{memory_context}" if preserved else memory_context
+
+
 class Logger(Protocol):
     """Logger protocol for type safety."""
 
@@ -231,25 +263,32 @@ def deduplicate_memories(
     static_memories: list[str] = []
     seen_memories: set[str] = set()
 
+    def normalize_fact(memory: str) -> str:
+        without_date = re.sub(r"^\[\d{4}-\d{2}-\d{2}\]\s*", "", memory)
+        return " ".join(without_date.strip().split()).casefold()
+
     for item in static_items:
         memory = extract_memory_text(item)
-        if memory is not None:
+        key = normalize_fact(memory) if memory is not None else None
+        if memory is not None and key is not None and key not in seen_memories:
             static_memories.append(memory)
-            seen_memories.add(memory)
+            seen_memories.add(key)
 
     dynamic_memories: list[str] = []
     for item in dynamic_items:
         memory = extract_memory_text(item)
-        if memory is not None and memory not in seen_memories:
+        key = normalize_fact(memory) if memory is not None else None
+        if memory is not None and key is not None and key not in seen_memories:
             dynamic_memories.append(memory)
-            seen_memories.add(memory)
+            seen_memories.add(key)
 
     search_memories: list[str] = []
     for item in search_items:
         memory = extract_memory_text(item)
-        if memory is not None and memory not in seen_memories:
+        key = normalize_fact(memory) if memory is not None else None
+        if memory is not None and key is not None and key not in seen_memories:
             search_memories.append(memory)
-            seen_memories.add(memory)
+            seen_memories.add(key)
 
     return DeduplicatedMemories(
         static=static_memories,

packages/openai-sdk-python/tests/test_middleware.py

@@ -215,7 +215,10 @@ class TestMemoryInjection:
         with patch.dict(os.environ, {"SUPERMEMORY_API_KEY": "test-key"}):
             with patch("supermemory_openai.middleware.supermemory_profile_search") as mock_search:
                 mock_search.return_value = Mock()
-                mock_search.return_value.profile = {"static": [], "dynamic": []}
+                mock_search.return_value.profile = {
+                    "static": [{"memory": "User likes machine learning projects"}],
+                    "dynamic": [],
+                }
                 mock_search.return_value.search_results = mock_supermemory_response["searchResults"]
 
                 wrapped_client = with_supermemory(
@@ -236,6 +239,8 @@ class TestMemoryInjection:
                 mock_search.assert_called_once()
                 search_args = mock_search.call_args[0]
                 assert search_args[1] == "What machine learning frameworks do I like?"
+                enhanced_messages = original_create.call_args[1]["messages"]
+                assert "User likes machine learning projects" in enhanced_messages[0]["content"]
 
     @pytest.mark.asyncio
     async def test_memory_injection_full_mode(
@@ -295,7 +300,15 @@ class TestMemoryInjection:
                 )
 
                 messages = [
-                    {"role": "system", "content": "You are a helpful assistant."},
+                    {
+                        "role": "system",
+                        "content": (
+                            "You are a helpful assistant.\n\n"
+                            '<supermemory context="user-memories" readonly>\n'
+                            "Stale profile fact\n"
+                            "</supermemory>"
+                        ),
+                    },
                     {"role": "user", "content": "What do you know about me?"}
                 ]
 
@@ -316,6 +329,10 @@ class TestMemoryInjection:
                 assert system_message["role"] == "system"
                 assert "You are a helpful assistant." in system_message["content"]
                 assert "User prefers Python" in system_message["content"]
+                assert "Stale profile fact" not in system_message["content"]
+                assert system_message["content"].count(
+                    '<supermemory context="user-memories" readonly>'
+                ) == 1
 
 
     @pytest.mark.asyncio
@@ -794,4 +811,4 @@ class TestBackgroundTaskManagement:
                             messages=[{"role": "user", "content": "Hello"}]
                         )
 
-                    # Should complete without error
+                    # Should complete without error

packages/openai-sdk-python/tests/test_utils.py

@@ -0,0 +1,17 @@
+"""Tests for shared middleware utilities."""
+
+from supermemory_openai.utils import deduplicate_memories
+
+
+def test_deduplicates_normalized_fact_variants() -> None:
+    result = deduplicate_memories(
+        static=[
+            {"memory": "User likes Python"},
+            {"memory": " user likes python "},
+        ],
+        dynamic=[{"memory": "[2026-08-10] USER LIKES PYTHON"}],
+        search_results=[],
+    )
+
+    assert result.static == ["User likes Python"]
+    assert result.dynamic == []

packages/pipecat-sdk-python/src/supermemory_pipecat/service.py

@@ -359,9 +359,11 @@ class SupermemoryPipecatService(FrameProcessor):
             memories_data: Memory data from Supermemory API.
         """
         profile = memories_data["profile"]
+        include_profile = self.params.mode in ("profile", "full")
+        include_search = self.params.mode in ("query", "full")
         deduplicated = deduplicate_memories(
-            static=profile["static"],
-            dynamic=profile["dynamic"],
+            static=profile["static"] if include_profile else [],
+            dynamic=profile["dynamic"] if include_profile else [],
             search_results=memories_data["search_results"],
         )
 
@@ -374,9 +376,6 @@ class SupermemoryPipecatService(FrameProcessor):
         if total_memories == 0:
             return
 
-        include_profile = self.params.mode in ("profile", "full")
-        include_search = self.params.mode in ("query", "full")
-
         memory_text = format_memories_to_text(
             deduplicated,
             system_prompt=self.params.system_prompt,

packages/pipecat-sdk-python/src/supermemory_pipecat/utils.py

@@ -5,7 +5,10 @@ from datetime import datetime, timezone
 from typing import Any, Dict, List, Union
 
 
-_DYNAMIC_DATE_PREFIX = re.compile(r"^\s*(?:\[Recent\]\s*)?\[\d{4}-\d{2}-\d{2}\]\s*")
+_DYNAMIC_DATE_PREFIX = re.compile(
+    r"^\s*(?:\[Recent\]\s*)?\[\d{4}-\d{2}-\d{2}\]\s*",
+    re.IGNORECASE,
+)
 
 
 def get_last_user_message(messages: List[Dict[str, Any]]) -> str | None:
@@ -91,7 +94,8 @@ def deduplicate_memories(
     def comparison_key(memory: str) -> str:
         # Dynamic profile entries are date-labelled by the API while search
         # results contain the same memory without that presentation prefix.
-        return _DYNAMIC_DATE_PREFIX.sub("", memory.strip())
+        without_prefix = _DYNAMIC_DATE_PREFIX.sub("", memory.strip())
+        return " ".join(without_prefix.split()).casefold()
 
     def unique_strings(memories: List[str]) -> List[str]:
         out: List[str] = []

packages/pipecat-sdk-python/tests/test_empty_profile.py

@@ -120,4 +120,37 @@ class TestSupermemoryPipecatNullProfile(unittest.IsolatedAsyncioTestCase):
                 "profile": {"static": [], "dynamic": []},
                 "search_results": [],
             },
-        )
+        )
+
+    def test_query_mode_keeps_search_fact_also_present_in_profile(self) -> None:
+        fact = "User likes machine learning projects"
+        service = SupermemoryPipecatService(
+            api_key="mock_key",
+            user_id="user-123",
+            session_id="conversation-456",
+            params=SupermemoryPipecatService.InputParams(mode="query"),
+        )
+
+        class Context:
+            def __init__(self):
+                self.messages = [{"role": "user", "content": "What do I like?"}]
+
+            def get_messages(self):
+                return self.messages
+
+            def add_message(self, message):
+                self.messages.append(message)
+
+        context = Context()
+        service._enhance_context_with_memories(
+            context,
+            "What do I like?",
+            {
+                "profile": {"static": [fact], "dynamic": []},
+                "search_results": [SimpleNamespace(memory=fact)],
+            },
+        )
+
+        self.assertTrue(
+            any(fact in message.get("content", "") for message in context.messages)
+        )