ReMe/tests4/integration/test_auto_resource.py

"""Integration test for the auto_resource job.

Drives the ``auto_resource`` step against a real LLM. Three scenarios:

1. **CREATE (added)** / **UPDATE (modified)**: places a resource file in
   ``resource/{date}/``, calls ``auto_resource`` with a ``changes`` batch,
   and expects the agent to write/update the same-name daily note.

2. **DELETE (deleted)**: seeds a resource note under
   ``daily/{date}/{resource_stem}.md``, calls ``auto_resource`` with a
   deleted change, and expects the note file to be removed.

Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the
environment or a .env file at the repo root. Hits the real LLM API.
"""

import asyncio
import sys
from pathlib import Path

INTEGRATION_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(INTEGRATION_DIR))

# pylint: disable=wrong-import-position
from _vault_fixture import vault_env  # noqa: E402

from reme4.steps.evolve.auto_resource import _compute_agent_session_id, _compute_note_stem  # noqa: E402

RESOURCE_FILENAME = "project-roadmap.md"
RESOURCE_CONTENT_V1 = """\
# Project Roadmap 2026 Q3

## Goals
- Launch v2.0 API by July 15
- Migrate 80% of users to new auth system by August 1
- Reduce p99 latency to < 200ms

## Milestones
| Date       | Milestone              | Owner   |
|------------|------------------------|---------|
| 2026-07-01 | API beta release       | Alice   |
| 2026-07-15 | API GA                 | Alice   |
| 2026-08-01 | Auth migration done    | Bob     |
| 2026-08-15 | Performance target met | Charlie |

## Risks
- Auth migration blocked on legacy client deprecation (ETA: June 30)
- Performance target requires Redis cluster upgrade (budget approved)
"""

RESOURCE_CONTENT_V2 = """\
# Project Roadmap 2026 Q3 (Revised)

## Goals
- Launch v2.0 API by July 20 (delayed 5 days from original July 15)
- Migrate 80% of users to new auth system by August 1
- Reduce p99 latency to < 150ms (tightened from 200ms)

## Milestones
| Date       | Milestone              | Owner   |
|------------|------------------------|---------|
| 2026-07-05 | API beta release       | Alice   |
| 2026-07-20 | API GA                 | Alice   |
| 2026-08-01 | Auth migration done    | Bob     |
| 2026-08-15 | Performance target met | Charlie |
| 2026-08-20 | Post-launch review     | Dave    |

## Risks
- Auth migration blocked on legacy client deprecation (resolved June 28)
- Performance target requires Redis cluster upgrade (completed July 1)
- New risk: third-party OAuth provider rate limiting during migration
"""


def _read_text(p: Path) -> str:
    return p.read_text(encoding="utf-8")


def _print_text_file(label: str, path: Path) -> str:
    text = _read_text(path)
    print("\n" + "=" * 70)
    print(f"[{label}] {path} ({len(text)} bytes)")
    print(f"[{label}] body:\n{text}")
    print("=" * 70)
    return text


def _print_message_files(label: str, paths: list[Path]) -> None:
    for idx, path in enumerate(paths, 1):
        if not path.is_file():
            print(f"[{label}] message file missing: {path}")
            continue
        text = _read_text(path)
        print("\n" + "=" * 70)
        print(f"[{label}] message {idx}: {path} ({len(text)} bytes)")
        print(text)
        print("=" * 70)


def test_auto_resource_create():
    """CREATE branch: agent writes the same-name daily note and saves its AgentScope session."""

    async def run():
        with vault_env() as env:
            app = await env.make_app()
            try:
                today = env.today

                print("\n" + "=" * 70)
                print("[setup] vault_root =", env.vault_dir)
                print("[setup] today      =", today)
                print("=" * 70)

                file_path = env.place_resource(RESOURCE_FILENAME, RESOURCE_CONTENT_V1)
                note_stem = _compute_note_stem(RESOURCE_FILENAME)
                agent_session_id = _compute_agent_session_id(file_path)
                expected_session_jsonl = env.vault_dir / "reme_session" / "agentscope" / f"{agent_session_id}.jsonl"

                print(f"[CREATE] file_path           = {file_path}")
                print(f"[CREATE] note_stem           = {note_stem}")
                print(f"[CREATE] agent_session_id    = {agent_session_id}")
                print(f"[CREATE] expected transcript = {expected_session_jsonl.relative_to(env.vault_dir)}")

                with env.record_agents(prefix="agent_resource_create") as recorder:
                    response = await app.run_job(
                        "auto_resource",
                        changes=[{"path": file_path, "change": "added"}],
                    )
                dumped = await recorder.dump()
                for p in dumped:
                    print(f"[CREATE] agent memory dumped: {p}")

                assert response.success is True, f"CREATE job failed: {response.answer!r}"
                meta = response.metadata or {}
                result_meta = (meta.get("results") or [{}])[0].get("metadata") or {}
                assert result_meta.get("action") == "added", f"Unexpected action: {meta!r}"
                assert result_meta.get("session_id") == note_stem, f"Unexpected session_id: {meta!r}"
                assert result_meta.get("path") == f"daily/{today}/{note_stem}.md", f"Unexpected note path: {meta!r}"
                note_path = env.vault_dir / "daily" / today / f"{note_stem}.md"
                assert note_path.is_file()

                assert expected_session_jsonl.is_file(), (
                    f"agent session not persisted at {expected_session_jsonl}; "
                    f"AgentScope files: "
                    f"{[p.name for p in (env.vault_dir / 'reme_session' / 'agentscope').glob('*.jsonl')]}"
                )

                note_text = _print_text_file("CREATE result.md", note_path)
                _print_message_files("CREATE intermediate messages", [*dumped, expected_session_jsonl])

                note_hits = [
                    needle
                    for needle in ("v2.0", "July 15", "Alice", "Bob", "p99", "200ms", "Redis")
                    if needle in note_text
                ]
                print(f"[CREATE] landed note facts: {note_hits}")
                assert (
                    len(note_hits) >= 3
                ), f"CREATE note missed expected facts {note_hits!r}\n--- NOTE ---\n{note_text}"

                transcript = _read_text(expected_session_jsonl)
                topic_hits = [
                    needle
                    for needle in ("v2.0", "July 15", "Alice", "Bob", "p99", "200ms", "Redis", file_path)
                    if needle in transcript
                ]
                print(f"[CREATE] facts visible in transcript: {topic_hits}")
                assert topic_hits, (
                    "agent transcript shows no signal it actually read the resource file; "
                    f"transcript head:\n{transcript[:500]}"
                )

                print("\n" + "=" * 70)
                print("test_auto_resource_create passed")
                print("=" * 70)
            finally:
                await env.close_all()

    asyncio.run(run())


def test_auto_resource_update():
    """UPDATE branch: agent updates the same-name daily note and appends to its AgentScope session."""

    async def run():
        with vault_env() as env:
            app = await env.make_app()
            try:
                today = env.today

                print("\n" + "=" * 70)
                print("[setup] vault_root =", env.vault_dir)
                print("[setup] today      =", today)
                print("=" * 70)

                # First run as "added" so the resource file exists and the
                # initial transcript lands.
                file_path = env.place_resource(RESOURCE_FILENAME, RESOURCE_CONTENT_V1)
                note_stem = _compute_note_stem(RESOURCE_FILENAME)
                agent_session_id = _compute_agent_session_id(file_path)
                session_jsonl = env.vault_dir / "reme_session" / "agentscope" / f"{agent_session_id}.jsonl"

                response = await app.run_job("auto_resource", changes=[{"path": file_path, "change": "added"}])
                assert response.success is True, f"Initial create failed: {response.answer!r}"
                assert session_jsonl.is_file(), "initial added run did not save the AgentScope session"
                size_before = session_jsonl.stat().st_size
                print(f"[UPDATE] transcript before modify ({size_before} bytes)")

                # Now update the resource file and call with "modified".
                env.place_resource(RESOURCE_FILENAME, RESOURCE_CONTENT_V2)

                with env.record_agents(prefix="agent_resource_update") as recorder:
                    response = await app.run_job(
                        "auto_resource",
                        changes=[{"path": file_path, "change": "modified"}],
                    )
                dumped = await recorder.dump()
                for p in dumped:
                    print(f"[UPDATE] agent memory dumped: {p}")

                assert response.success is True, f"UPDATE job failed: {response.answer!r}"
                meta = response.metadata or {}
                result_meta = (meta.get("results") or [{}])[0].get("metadata") or {}
                assert result_meta.get("action") == "modified", f"Unexpected action: {meta!r}"
                assert result_meta.get("session_id") == note_stem, f"Unexpected session_id: {meta!r}"
                assert result_meta.get("path") == f"daily/{today}/{note_stem}.md", f"Unexpected note path: {meta!r}"
                note_path = env.vault_dir / "daily" / today / f"{note_stem}.md"
                assert note_path.is_file()

                size_after = session_jsonl.stat().st_size
                print(f"[UPDATE] transcript after modify  ({size_after} bytes)")
                assert size_after > size_before, (
                    f"transcript did not grow after modified run " f"({size_before} -> {size_after})"
                )

                note_text = _print_text_file("UPDATE result.md", note_path)
                _print_message_files("UPDATE intermediate messages", [*dumped, session_jsonl])

                note_hits = [
                    needle
                    for needle in ("July 20", "150ms", "Dave", "rate limiting", "resolved")
                    if needle in note_text
                ]
                print(f"[UPDATE] landed note facts: {note_hits}")
                assert (
                    len(note_hits) >= 2
                ), f"UPDATE note missed expected facts {note_hits!r}\n--- NOTE ---\n{note_text}"

                transcript = _read_text(session_jsonl)
                new_hits = [
                    needle
                    for needle in ("July 20", "150ms", "Dave", "rate limiting", "resolved")
                    if needle in transcript
                ]
                print(f"[UPDATE] V2 facts visible in transcript: {new_hits}")
                assert new_hits, "modified run added no V2 content to the transcript; " f"tail:\n{transcript[-800:]}"

                print("\n" + "=" * 70)
                print("test_auto_resource_update passed")
                print("=" * 70)
            finally:
                await env.close_all()

    asyncio.run(run())


def test_auto_resource_delete():
    """DELETE a resource note (change=deleted)."""

    async def run():
        with vault_env() as env:
            app = await env.make_app()
            try:
                today = env.today

                print("\n" + "=" * 70)
                print("[setup] vault_root =", env.vault_dir)
                print("[setup] today      =", today)
                print("=" * 70)

                note_stem = _compute_note_stem(RESOURCE_FILENAME)
                file_path = f"resource/{today}/{RESOURCE_FILENAME}"

                seed_body = "---\nname: test\ndescription: test note\n---\n\nSome content.\n"
                note_path = env.seed_daily_note(note_stem, seed_body)
                assert note_path.is_file()
                print(f"[DELETE] seeded note: {note_path}")

                response = await app.run_job(
                    "auto_resource",
                    changes=[{"path": file_path, "change": "deleted"}],
                )

                assert response.success is True, f"DELETE job failed: {response.answer!r}"
                meta = response.metadata or {}
                result_meta = (meta.get("results") or [{}])[0].get("metadata") or {}
                assert result_meta.get("action") == "deleted"
                assert not note_path.is_file(), f"Note file still exists after delete: {note_path}"

                print(f"[DELETE] note removed: {note_path}")
                print("\n" + "=" * 70)
                print("test_auto_resource_delete passed")
                print("=" * 70)
            finally:
                await env.close_all()

    asyncio.run(run())


if __name__ == "__main__":
    print("=== auto_resource integration test ===")
    test_auto_resource_create()
    test_auto_resource_update()
    test_auto_resource_delete()
    print("\nAll integration tests passed!")