fix(roi): preserve reports during metadata outages

litellm/proxy/roi_calculator/sync.py

@@ -4,7 +4,7 @@ from contextlib import suppress
 from datetime import date, datetime, timedelta, timezone
 from itertools import chain
 from types import MappingProxyType
-from typing import Final, Literal, Protocol, runtime_checkable
+from typing import Final, Literal, NamedTuple, Protocol, runtime_checkable
 from uuid import uuid4
 
 import httpx
@@ -239,6 +239,12 @@ async def _unavailable_record(github: GitHub, repo: str, pull: GitHubPullListIte
     )
 
 
+class _ProcessedPull(NamedTuple):
+    position: int
+    record: ROIPullRecord
+    metadata_unavailable: bool = False
+
+
 class SyncManager:
     def __init__(
         self,
@@ -398,7 +404,7 @@ class SyncManager:
 
             async def process(
                 item: tuple[int, str, GitHubPullListItem, str | None],
-            ) -> tuple[int, ROIPullRecord]:
+            ) -> _ProcessedPull:
                 index, repo, pull, key = item
                 saved: Final = await repository.get_param("roi_calculator_pull_" + key) if key is not None else None
                 cached_pull: Final = (
@@ -426,13 +432,13 @@ class SyncManager:
                         )
                     )
                     self._update_estimate_progress(cached_record["estimate"])
-                    return index, cached_record
+                    return _ProcessedPull(index, cached_record)
                 try:
                     evidence: Final = await github.evidence(repo, pull)
                 except SourceError as exc:
                     unavailable: Final = await _unavailable_record(github, repo, pull, exc)
                     self._update_estimate_progress(unavailable["estimate"])
-                    return index, unavailable
+                    return _ProcessedPull(index, unavailable, metadata_unavailable=True)
                 estimate: Final = await _estimate_with_fallback(estimator, evidence)
                 evidence_item: Final = GitHubPullListItem.model_validate(
                     MappingProxyType(
@@ -457,9 +463,9 @@ class SyncManager:
                         ),
                     )
                 self._update_estimate_progress(estimate)
-                return index, record
+                return _ProcessedPull(index, record)
 
-            async def worker(offset: int) -> tuple[tuple[int, ROIPullRecord], ...]:
+            async def worker(offset: int) -> tuple[_ProcessedPull, ...]:
                 return tuple(
                     [await process(indexed_queue[index]) for index in range(offset, len(indexed_queue), PR_CONCURRENCY)]
                 )
@@ -473,8 +479,12 @@ class SyncManager:
                     if not worker_task.done():
                         worker_task.cancel()
                 await asyncio.gather(*workers, return_exceptions=True)
+            if processed and all(item.metadata_unavailable for item in processed):
+                raise SourceError(
+                    "GitHub could not provide PR metadata. No new report was published; try analysis again later."
+                )
             processed_by_index: Final[Mapping[int, ROIPullRecord]] = MappingProxyType(
-                {index: pull for index, pull in processed}
+                {item.position: item.record for item in processed}
             )
             report: Final = ROIReport(
                 mode="live",

tests/unit/proxy/roi_calculator/test_sync.py

@@ -275,12 +275,15 @@ async def test_read_spend_joins_user_emails_and_preserves_unmatched_identities()
 
 
 @pytest.mark.asyncio
-async def test_unreadable_pr_is_reported_and_retried_on_next_run() -> None:
+async def test_metadata_outage_keeps_previous_report_and_retries_on_next_run() -> None:
     repository: Final = _ReportRepository()
     manager: Final = SyncManager(clock=_fixed_now)
 
+    assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
+    await _wait_until_finished(manager)
+    previous: Final = repository.values["roi_calculator_report"]
     assert await manager.start(
-        _settings(),
+        _settings(estimator_prompt="New prompt invalidates saved estimates"),
         repository,
         _spend_reader(),
         _completion(),
@@ -288,19 +291,16 @@ async def test_unreadable_pr_is_reported_and_retried_on_next_run() -> None:
     )
     await _wait_until_finished(manager)
 
-    assert manager.status.phase == "complete"
+    assert manager.status.phase == "error"
     assert manager.status.needs_attention == 1
-    failed: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
-    assert failed["pulls"][0]["estimate"]["status"] == "needs_review"
-    assert failed["pulls"][0]["estimate"]["hours"] is None
-    assert failed["pulls"][0]["incomplete_metadata"] is True
-    assert failed["pulls"][0]["cache_key"] is None
+    assert manager.status.error is not None and "No new report was published" in manager.status.error
+    assert repository.values["roi_calculator_report"] == previous
     assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
     await _wait_until_finished(manager)
     recovered: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
     assert recovered["pulls"][0]["estimate"]["status"] == "estimated"
     assert recovered["pulls"][0]["estimate"]["hours"] == 4
-    assert manager.status.reused == 0
+    assert manager.status.reused == 1
 
 
 @pytest.mark.asyncio

ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import React from "react";
-import { BarChart3, RefreshCw } from "lucide-react";
+import { Calculator, RefreshCw } from "lucide-react";
 
 import { apiClient } from "@/components/networking";
 import { PageHeader } from "@/components/shared/PageHeader";
@@ -220,7 +220,7 @@ export default function ROICalculatorView({
   return (
     <main className="w-full space-y-6 p-8">
       <PageHeader
-        icon={<BarChart3 />}
+        icon={<Calculator />}
         title="ROI Calculator"
         subtitle={
           <>