mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-19 00:01:29 +00:00
fix(memory): preserve round directives and component route discovery
This commit is contained in:
parent
f40f3cb677
commit
39b98fdb89
5 changed files with 54 additions and 1079 deletions
|
|
@ -56,11 +56,6 @@ class LazyFeature:
|
|||
|
||||
|
||||
LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
||||
LazyFeature(
|
||||
name="memory_v2",
|
||||
module_path="litellm.proxy.memory.management",
|
||||
path_prefixes=("/v2/memory",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="guardrails",
|
||||
module_path="litellm.proxy.guardrails.guardrail_endpoints",
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -137,8 +137,21 @@ class GatewayMemoryLoop:
|
|||
|
||||
async def _call(self) -> AsyncGenerator[bytes, None]:
|
||||
self.stream.begin_round()
|
||||
# Claude output directives control the next generated turn. Repeat them
|
||||
# on outgoing rounds without adding pending directives to saved history.
|
||||
directives: Final = (
|
||||
transcript_items(self.original, self.route)[-self.replaced_input :] if self.replaced_input else ()
|
||||
)
|
||||
messages: Final = transcript_items(self.data, self.route)
|
||||
body: Final = { # mutable-ok: Native provider JSON containers.
|
||||
**self.data,
|
||||
**(
|
||||
{ # mutable-ok: Native provider JSON containers.
|
||||
"messages": [*messages, *directives], # mutable-ok: Provider request JSON.
|
||||
}
|
||||
if directives and messages[-len(directives) :] != directives
|
||||
else {} # mutable-ok: Native provider JSON containers.
|
||||
),
|
||||
"cache": { # mutable-ok: Native provider JSON containers.
|
||||
**object_value(self.data.get("cache")),
|
||||
"no-cache": True,
|
||||
|
|
|
|||
|
|
@ -18802,6 +18802,9 @@ app.include_router(auto_router_management_router)
|
|||
app.include_router(tag_management_router)
|
||||
app.include_router(workflow_management_router)
|
||||
app.include_router(memory_router)
|
||||
from litellm.proxy.memory.management import router as memory_v2_router
|
||||
|
||||
app.include_router(memory_v2_router)
|
||||
app.include_router(plugin_router)
|
||||
app.include_router(cost_tracking_settings_router)
|
||||
app.include_router(router_settings_router)
|
||||
|
|
|
|||
|
|
@ -579,6 +579,44 @@ async def test_trailing_system_messages_survive_client_tool_continuation(prisma_
|
|||
assert original["messages"] == [prefix, directive]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_output_directives_reach_search_answer_and_reflection_rounds(prisma_edge: MagicMock) -> None:
|
||||
provider = FastAPI()
|
||||
observed = []
|
||||
directive = {"role": "system", "content": [], "output_config": {"effort": "low"}}
|
||||
|
||||
@provider.post("/v1/messages")
|
||||
async def model(incoming: Request):
|
||||
body = await incoming.json()
|
||||
observed.append(body)
|
||||
assert body["messages"][-1] == directive
|
||||
if len(observed) == 1:
|
||||
content = [
|
||||
{"type": "tool_use", "id": "search", "name": "litellm_memory_search", "input": {"query": "demo"}}
|
||||
]
|
||||
elif len(observed) == 2:
|
||||
content = [{"type": "text", "text": "The port is 8347"}]
|
||||
else:
|
||||
content = [
|
||||
{"type": "tool_use", "id": "reflect", "name": "litellm_memory_capture", "input": {"observations": []}}
|
||||
]
|
||||
return {
|
||||
"id": "msg_" + str(len(observed)),
|
||||
"stop_reason": "end_turn" if len(observed) == 2 else "tool_use",
|
||||
"content": content,
|
||||
}
|
||||
|
||||
original = {"messages": [{"role": "user", "content": "My demo port?"}, directive]}
|
||||
loop = GatewayMemoryLoop(provider, request(), original, "anthropic_messages", store(prisma_edge))
|
||||
async for _ in loop.run():
|
||||
pass
|
||||
assert len(observed) == 3
|
||||
assert observed[1]["messages"][-2]["content"][0]["tool_use_id"] == "search"
|
||||
assert "reflect once" in observed[2]["messages"][-2]["content"]
|
||||
assert observed[0]["messages"][0] == original["messages"][0]
|
||||
assert sum(message["role"] == "system" for message in object_items(loop.data.get("messages"))) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("bad_id,count", [(True, 1), (False, 17)])
|
||||
async def test_invalid_model_calls_are_rejected_before_storage(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue