fix(memory): preserve round directives and component route discovery

This commit is contained in:
moe-berri 2026-09-14 14:05:26 -07:00
parent f40f3cb677
commit 39b98fdb89
5 changed files with 54 additions and 1079 deletions

View file

@ -56,11 +56,6 @@ class LazyFeature:
LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
LazyFeature(
name="memory_v2",
module_path="litellm.proxy.memory.management",
path_prefixes=("/v2/memory",),
),
LazyFeature(
name="guardrails",
module_path="litellm.proxy.guardrails.guardrail_endpoints",

File diff suppressed because it is too large Load diff

View file

@ -137,8 +137,21 @@ class GatewayMemoryLoop:
async def _call(self) -> AsyncGenerator[bytes, None]:
self.stream.begin_round()
# Claude output directives control the next generated turn. Repeat them
# on outgoing rounds without adding pending directives to saved history.
directives: Final = (
transcript_items(self.original, self.route)[-self.replaced_input :] if self.replaced_input else ()
)
messages: Final = transcript_items(self.data, self.route)
body: Final = { # mutable-ok: Native provider JSON containers.
**self.data,
**(
{ # mutable-ok: Native provider JSON containers.
"messages": [*messages, *directives], # mutable-ok: Provider request JSON.
}
if directives and messages[-len(directives) :] != directives
else {} # mutable-ok: Native provider JSON containers.
),
"cache": { # mutable-ok: Native provider JSON containers.
**object_value(self.data.get("cache")),
"no-cache": True,

View file

@ -18802,6 +18802,9 @@ app.include_router(auto_router_management_router)
app.include_router(tag_management_router)
app.include_router(workflow_management_router)
app.include_router(memory_router)
from litellm.proxy.memory.management import router as memory_v2_router
app.include_router(memory_v2_router)
app.include_router(plugin_router)
app.include_router(cost_tracking_settings_router)
app.include_router(router_settings_router)

View file

@ -579,6 +579,44 @@ async def test_trailing_system_messages_survive_client_tool_continuation(prisma_
assert original["messages"] == [prefix, directive]
@pytest.mark.asyncio
async def test_claude_output_directives_reach_search_answer_and_reflection_rounds(prisma_edge: MagicMock) -> None:
provider = FastAPI()
observed = []
directive = {"role": "system", "content": [], "output_config": {"effort": "low"}}
@provider.post("/v1/messages")
async def model(incoming: Request):
body = await incoming.json()
observed.append(body)
assert body["messages"][-1] == directive
if len(observed) == 1:
content = [
{"type": "tool_use", "id": "search", "name": "litellm_memory_search", "input": {"query": "demo"}}
]
elif len(observed) == 2:
content = [{"type": "text", "text": "The port is 8347"}]
else:
content = [
{"type": "tool_use", "id": "reflect", "name": "litellm_memory_capture", "input": {"observations": []}}
]
return {
"id": "msg_" + str(len(observed)),
"stop_reason": "end_turn" if len(observed) == 2 else "tool_use",
"content": content,
}
original = {"messages": [{"role": "user", "content": "My demo port?"}, directive]}
loop = GatewayMemoryLoop(provider, request(), original, "anthropic_messages", store(prisma_edge))
async for _ in loop.run():
pass
assert len(observed) == 3
assert observed[1]["messages"][-2]["content"][0]["tool_use_id"] == "search"
assert "reflect once" in observed[2]["messages"][-2]["content"]
assert observed[0]["messages"][0] == original["messages"][0]
assert sum(message["role"] == "system" for message in object_items(loop.data.get("messages"))) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("bad_id,count", [(True, 1), (False, 17)])
async def test_invalid_model_calls_are_rejected_before_storage(