feat(lens): append worker reviews to the job, capped, and count every review

litellm/proxy/lens/endpoints.py

@@ -46,6 +46,7 @@ from litellm.proxy.lens.models import (
 from litellm.proxy.lens.repository import LensRepository, WriterDatabase
 from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
 from litellm.proxy.lens.state import (
+    add_review,
     add_step,
     can_access,
     claim_job,
@@ -490,10 +491,13 @@ async def progress(lens_id: str, job_id: str, body: Progress, worker: WorkerAuth
         job: Final = current_job(e)
         if job is None or job.id != job_id or job.worker_id != worker.id:
             return e
-        renewed: Final = job.model_copy(
-            update=MappingProxyType(
-                {"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
-            )
+        renewed: Final = add_review(
+            job.model_copy(
+                update=MappingProxyType(
+                    {"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
+                )
+            ),
+            body.review,
         )
         return replace_job(
             e,

litellm/proxy/lens/state.py

@@ -4,12 +4,14 @@ from types import MappingProxyType
 from typing import Final, Literal
 
 from litellm.proxy.lens.models import (
+    MAX_REVIEWS,
     MAX_STEPS,
     Finding,
     FindingDraft,
     Job,
     Lens,
     LensSettings,
+    Review,
     Scope,
     Step,
     Worker,
@@ -82,6 +84,14 @@ def add_step(job: Job, step: Step) -> Job:
     return job.model_copy(update=MappingProxyType({"steps": (*job.steps, step)[-MAX_STEPS:]}))
 
 
+def add_review(job: Job, review: Review | None) -> Job:
+    if review is None:
+        return job
+    return job.model_copy(
+        update=MappingProxyType({"reviews": (*job.reviews, review)[-MAX_REVIEWS:], "reviewed": job.reviewed + 1})
+    )
+
+
 def claim_job(lens: Lens, worker: Worker, now: datetime) -> Lens:
     job: Final = current_job(lens)
     if job is None or not can_access(worker.scope, lens.scope):

tests/unit/proxy/lens/test_state.py

@@ -5,6 +5,7 @@ from typing import Final
 import pytest
 
 from litellm.proxy.lens.models import (
+    MAX_REVIEWS,
     MAX_STEPS,
     AgentTestCase,
     Check,
@@ -13,11 +14,13 @@ from litellm.proxy.lens.models import (
     IssueBrief,
     Lens,
     LensSettings,
+    Review,
     Scope,
     Step,
     Worker,
 )
 from litellm.proxy.lens.state import (
+    add_review,
     add_step,
     can_access,
     claim_job,
@@ -345,3 +348,23 @@ def test_calendar_overflow_is_rejected_without_the_old_history_and_interval_caps
     assert getattr(accepted, field) == 100000
     with pytest.raises(ValidationError, match="supported calendar range"):
         LensSettings.model_validate({**lens().settings.model_dump(), field: 10**30})
+
+
+def review(index: int) -> Review:
+    return Review(
+        execution_id=f"run-{index}", trace_id="t", agent="support", name="task", model="analysis", duration_ms=1, at=NOW
+    )
+
+
+def test_reviews_keep_the_newest_window_while_counting_every_review() -> None:
+    job: Final = queue_job(lens(), NOW, "job").jobs[0]
+    grown: Final = reduce(add_review, tuple(review(i) for i in range(MAX_REVIEWS + 3)), job)
+    assert grown.reviewed == MAX_REVIEWS + 3
+    assert len(grown.reviews) == MAX_REVIEWS
+    assert grown.reviews[0].execution_id == "run-3"
+    assert grown.reviews[-1].execution_id == f"run-{MAX_REVIEWS + 2}"
+
+
+def test_progress_without_a_review_leaves_the_review_history_alone() -> None:
+    job: Final = add_review(queue_job(lens(), NOW, "job").jobs[0], review(0))
+    assert add_review(job, None) == job