diff --git a/.github/assets/roi-calculator/19-matching-calculator-icons.png b/.github/assets/roi-calculator/19-matching-calculator-icons.png new file mode 100644 index 00000000000..af12106e315 Binary files /dev/null and b/.github/assets/roi-calculator/19-matching-calculator-icons.png differ diff --git a/litellm/proxy/roi_calculator/sync.py b/litellm/proxy/roi_calculator/sync.py index 645d9ea60a8..bd909c9a036 100644 --- a/litellm/proxy/roi_calculator/sync.py +++ b/litellm/proxy/roi_calculator/sync.py @@ -4,7 +4,7 @@ from contextlib import suppress from datetime import date, datetime, timedelta, timezone from itertools import chain from types import MappingProxyType -from typing import Final, Literal, Protocol, runtime_checkable +from typing import Final, Literal, NamedTuple, Protocol, runtime_checkable from uuid import uuid4 import httpx @@ -239,6 +239,12 @@ async def _unavailable_record(github: GitHub, repo: str, pull: GitHubPullListIte ) +class _ProcessedPull(NamedTuple): + position: int + record: ROIPullRecord + metadata_unavailable: bool = False + + class SyncManager: def __init__( self, @@ -398,7 +404,7 @@ class SyncManager: async def process( item: tuple[int, str, GitHubPullListItem, str | None], - ) -> tuple[int, ROIPullRecord]: + ) -> _ProcessedPull: index, repo, pull, key = item saved: Final = await repository.get_param("roi_calculator_pull_" + key) if key is not None else None cached_pull: Final = ( @@ -426,13 +432,13 @@ class SyncManager: ) ) self._update_estimate_progress(cached_record["estimate"]) - return index, cached_record + return _ProcessedPull(index, cached_record) try: evidence: Final = await github.evidence(repo, pull) except SourceError as exc: unavailable: Final = await _unavailable_record(github, repo, pull, exc) self._update_estimate_progress(unavailable["estimate"]) - return index, unavailable + return _ProcessedPull(index, unavailable, metadata_unavailable=True) estimate: Final = await _estimate_with_fallback(estimator, evidence) evidence_item: Final = GitHubPullListItem.model_validate( MappingProxyType( @@ -457,9 +463,9 @@ class SyncManager: ), ) self._update_estimate_progress(estimate) - return index, record + return _ProcessedPull(index, record) - async def worker(offset: int) -> tuple[tuple[int, ROIPullRecord], ...]: + async def worker(offset: int) -> tuple[_ProcessedPull, ...]: return tuple( [await process(indexed_queue[index]) for index in range(offset, len(indexed_queue), PR_CONCURRENCY)] ) @@ -473,8 +479,12 @@ class SyncManager: if not worker_task.done(): worker_task.cancel() await asyncio.gather(*workers, return_exceptions=True) + if processed and all(item.metadata_unavailable for item in processed): + raise SourceError( + "GitHub could not provide PR metadata. No new report was published; try analysis again later." + ) processed_by_index: Final[Mapping[int, ROIPullRecord]] = MappingProxyType( - {index: pull for index, pull in processed} + {item.position: item.record for item in processed} ) report: Final = ROIReport( mode="live", diff --git a/tests/unit/proxy/roi_calculator/test_sync.py b/tests/unit/proxy/roi_calculator/test_sync.py index 9b80ae54e5a..4e827a7d1da 100644 --- a/tests/unit/proxy/roi_calculator/test_sync.py +++ b/tests/unit/proxy/roi_calculator/test_sync.py @@ -275,12 +275,15 @@ async def test_read_spend_joins_user_emails_and_preserves_unmatched_identities() @pytest.mark.asyncio -async def test_unreadable_pr_is_reported_and_retried_on_next_run() -> None: +async def test_metadata_outage_keeps_previous_report_and_retries_on_next_run() -> None: repository: Final = _ReportRepository() manager: Final = SyncManager(clock=_fixed_now) + assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport()) + await _wait_until_finished(manager) + previous: Final = repository.values["roi_calculator_report"] assert await manager.start( - _settings(), + _settings(estimator_prompt="New prompt invalidates saved estimates"), repository, _spend_reader(), _completion(), @@ -288,19 +291,16 @@ async def test_unreadable_pr_is_reported_and_retried_on_next_run() -> None: ) await _wait_until_finished(manager) - assert manager.status.phase == "complete" + assert manager.status.phase == "error" assert manager.status.needs_attention == 1 - failed: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"]) - assert failed["pulls"][0]["estimate"]["status"] == "needs_review" - assert failed["pulls"][0]["estimate"]["hours"] is None - assert failed["pulls"][0]["incomplete_metadata"] is True - assert failed["pulls"][0]["cache_key"] is None + assert manager.status.error is not None and "No new report was published" in manager.status.error + assert repository.values["roi_calculator_report"] == previous assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport()) await _wait_until_finished(manager) recovered: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"]) assert recovered["pulls"][0]["estimate"]["status"] == "estimated" assert recovered["pulls"][0]["estimate"]["hours"] == 4 - assert manager.status.reused == 0 + assert manager.status.reused == 1 @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx index 1d1bfca559c..1f5b136dbb7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx @@ -1,7 +1,7 @@ "use client"; import React from "react"; -import { BarChart3, RefreshCw } from "lucide-react"; +import { Calculator, RefreshCw } from "lucide-react"; import { apiClient } from "@/components/networking"; import { PageHeader } from "@/components/shared/PageHeader"; @@ -220,7 +220,7 @@ export default function ROICalculatorView({ return (
} + icon={} title="ROI Calculator" subtitle={ <>