diff --git a/.github/assets/roi-calculator-integrations/after-github.jpg b/.github/assets/roi-calculator-integrations/after-github.jpg new file mode 100644 index 00000000000..31789b9d309 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/after-github.jpg differ diff --git a/.github/assets/roi-calculator-integrations/after-gitlab-detail-top.jpg b/.github/assets/roi-calculator-integrations/after-gitlab-detail-top.jpg new file mode 100644 index 00000000000..6846edf14f7 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/after-gitlab-detail-top.jpg differ diff --git a/.github/assets/roi-calculator-integrations/after-gitlab-detail.jpg b/.github/assets/roi-calculator-integrations/after-gitlab-detail.jpg new file mode 100644 index 00000000000..2b8a541a4fb Binary files /dev/null and b/.github/assets/roi-calculator-integrations/after-gitlab-detail.jpg differ diff --git a/.github/assets/roi-calculator-integrations/after-gitlab.jpg b/.github/assets/roi-calculator-integrations/after-gitlab.jpg new file mode 100644 index 00000000000..792c2218353 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/after-gitlab.jpg differ diff --git a/.github/assets/roi-calculator-integrations/before-github.jpg b/.github/assets/roi-calculator-integrations/before-github.jpg new file mode 100644 index 00000000000..0154db63738 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/before-github.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-exit-loading.jpg b/.github/assets/roi-calculator-integrations/demo-exit-loading.jpg new file mode 100644 index 00000000000..3f194c59d82 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-exit-loading.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-fallback-live.jpg b/.github/assets/roi-calculator-integrations/demo-fallback-live.jpg new file mode 100644 index 00000000000..4e4788cdb75 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-fallback-live.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-overview.jpg b/.github/assets/roi-calculator-integrations/demo-overview.jpg new file mode 100644 index 00000000000..db5005aa1ae Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-overview.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-people.jpg b/.github/assets/roi-calculator-integrations/demo-people.jpg new file mode 100644 index 00000000000..b698fe500ac Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-people.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-pr-costs.jpg b/.github/assets/roi-calculator-integrations/demo-pr-costs.jpg new file mode 100644 index 00000000000..73c148d07e1 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-pr-costs.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-pr-detail.jpg b/.github/assets/roi-calculator-integrations/demo-pr-detail.jpg new file mode 100644 index 00000000000..7a6c321439d Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-pr-detail.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-preview-link.jpg b/.github/assets/roi-calculator-integrations/demo-preview-link.jpg new file mode 100644 index 00000000000..b4a1d0e8244 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-preview-link.jpg differ diff --git a/.github/assets/roi-calculator-integrations/demo-with-live-errors.jpg b/.github/assets/roi-calculator-integrations/demo-with-live-errors.jpg new file mode 100644 index 00000000000..41fc1320560 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/demo-with-live-errors.jpg differ diff --git a/.github/assets/roi-calculator-integrations/source-race-after.jpg b/.github/assets/roi-calculator-integrations/source-race-after.jpg new file mode 100644 index 00000000000..fac19265807 Binary files /dev/null and b/.github/assets/roi-calculator-integrations/source-race-after.jpg differ diff --git a/.github/assets/roi-calculator-integrations/source-race-before.jpg b/.github/assets/roi-calculator-integrations/source-race-before.jpg new file mode 100644 index 00000000000..d22ccd3deaa Binary files /dev/null and b/.github/assets/roi-calculator-integrations/source-race-before.jpg differ diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 727ecb235bb..5b3736d22a9 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -48960,6 +48960,121 @@ "title": "HTTPValidationError", "type": "object" }, + "ROIBranchAttribution": { + "properties": { + "branch": { + "title": "Branch", + "type": "string" + }, + "repo": { + "title": "Repo", + "type": "string" + }, + "requests": { + "default": 0, + "title": "Requests", + "type": "integer" + }, + "spend": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Spend" + }, + "status": { + "default": "unattributed", + "enum": [ + "matched", + "unattributed", + "ambiguous", + "unavailable" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "repo", + "branch" + ], + "title": "ROIBranchAttribution", + "type": "object" + }, + "ROIBranchMetrics": { + "properties": { + "cost_per_hour": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Per Hour" + }, + "hours": { + "default": 0, + "title": "Hours", + "type": "number" + }, + "matched_pulls": { + "default": 0, + "title": "Matched Pulls", + "type": "integer" + }, + "spend": { + "default": 0, + "title": "Spend", + "type": "number" + }, + "total_tagged_spend": { + "default": 0, + "title": "Total Tagged Spend", + "type": "number" + }, + "unlinked_spend": { + "default": 0, + "title": "Unlinked Spend", + "type": "number" + } + }, + "title": "ROIBranchMetrics", + "type": "object" + }, + "ROIBranchSpend": { + "properties": { + "branch": { + "title": "Branch", + "type": "string" + }, + "repo": { + "title": "Repo", + "type": "string" + }, + "requests": { + "title": "Requests", + "type": "integer" + }, + "spend": { + "title": "Spend", + "type": "number" + } + }, + "required": [ + "repo", + "branch", + "spend", + "requests" + ], + "title": "ROIBranchSpend", + "type": "object" + }, "ROIEstimateResponse": { "properties": { "cached": { @@ -49261,6 +49376,9 @@ "title": "Additions", "type": "integer" }, + "branch_cost": { + "$ref": "#/components/schemas/ROIBranchAttribution" + }, "cache_key": { "anyOf": [ { @@ -49334,6 +49452,16 @@ "title": "Repo", "type": "string" }, + "source_branch": { + "default": "", + "title": "Source Branch", + "type": "string" + }, + "source_repo": { + "default": "", + "title": "Source Repo", + "type": "string" + }, "title": { "title": "Title", "type": "string" @@ -49463,6 +49591,11 @@ "title": "Github Api Url", "type": "string" }, + "gitlab_api_url": { + "default": "https://gitlab.com/api/v4", + "title": "Gitlab Api Url", + "type": "string" + }, "has_estimator_key": { "title": "Has Estimator Key", "type": "boolean" @@ -49471,6 +49604,11 @@ "title": "Has Github Token", "type": "boolean" }, + "has_gitlab_token": { + "default": false, + "title": "Has Gitlab Token", + "type": "boolean" + }, "identity_map": { "additionalProperties": { "type": "string" @@ -49489,6 +49627,15 @@ "title": "Repos", "type": "array" }, + "source_provider": { + "default": "github", + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + }, "update_interval_minutes": { "title": "Update Interval Minutes", "type": "number" @@ -49582,6 +49729,28 @@ ], "title": "Github Token" }, + "gitlab_api_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Gitlab Api Url" + }, + "gitlab_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Gitlab Token" + }, "repos": { "anyOf": [ { @@ -49596,6 +49765,21 @@ ], "title": "Repos" }, + "source_provider": { + "anyOf": [ + { + "enum": [ + "github", + "gitlab" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Provider" + }, "update_interval_minutes": { "anyOf": [ { @@ -49615,6 +49799,9 @@ }, "ROISummaryResponse": { "properties": { + "branch_metrics": { + "$ref": "#/components/schemas/ROIBranchMetrics" + }, "effort_basis": { "anyOf": [ { @@ -49677,6 +49864,15 @@ "title": "Repos", "type": "array" }, + "source_provider": { + "default": "github", + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + }, "start": { "title": "Start", "type": "string" @@ -49692,6 +49888,14 @@ "title": "Trend", "type": "array" }, + "unlinked_branches": { + "default": [], + "items": { + "$ref": "#/components/schemas/ROIBranchSpend" + }, + "title": "Unlinked Branches", + "type": "array" + }, "warnings": { "items": { "type": "string" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 59302e9f08e..0abec51cc49 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4214,6 +4214,7 @@ class SpendLogsMetadata(TypedDict): vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None routing_decision: StandardLoggingRoutingDecision | None internal_call_origin: InternalCallOrigin | None + litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a809f53aa85..152438d0573 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -339,6 +339,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( ROUTING_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", + "litellm_roi_estimator", "proxy_server_request", "secret_fields", "_guardrail_pipelines", @@ -2565,6 +2566,10 @@ async def add_litellm_data_to_request( user_api_key_dict=user_api_key_dict, ) + data[_metadata_variable_name]["litellm_roi_estimator"] = ( + getattr(request.state, "litellm_roi_estimator", False) is True + ) + verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data) # Team/Project credential overrides from model_config diff --git a/litellm/proxy/management_endpoints/roi_calculator_endpoints.py b/litellm/proxy/management_endpoints/roi_calculator_endpoints.py index 7d214a7a075..728f41d7c2a 100644 --- a/litellm/proxy/management_endpoints/roi_calculator_endpoints.py +++ b/litellm/proxy/management_endpoints/roi_calculator_endpoints.py @@ -3,7 +3,12 @@ from datetime import date, datetime, timedelta, timezone from enum import Enum from functools import lru_cache from types import MappingProxyType -from typing import Annotated, Final, Literal +from typing import ( + Annotated, + Final, + Literal, + cast, # noqa: TID251 # PrismaWrapper dynamically delegates database methods +) import httpx from apscheduler.schedulers.asyncio import ( # pyright: ignore[reportMissingTypeStubs] # no upstream stubs @@ -11,6 +16,7 @@ from apscheduler.schedulers.asyncio import ( # pyright: ignore[reportMissingTyp ) from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError +from starlette.types import Receive, Scope, Send from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -20,13 +26,22 @@ from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKey from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper from litellm.proxy.roi_calculator.analytics import normalize_email, summarize +from litellm.proxy.roi_calculator.branch_spend import BranchSpendDatabase, read_branch_spend from litellm.proxy.roi_calculator.estimator import CompletionCaller, EstimatorModel -from litellm.proxy.roi_calculator.github import GitHub, SourceError -from litellm.proxy.roi_calculator.sync import SpendReader, SyncManager, read_spend, spend_prisma_client +from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.source import create_source +from litellm.proxy.roi_calculator.sync import ( + BranchSpendReader, + SpendReader, + SyncManager, + read_spend, + spend_prisma_client, +) from litellm.proxy.roi_calculator.sync_store import SyncStore from litellm.repositories.config_repository import ConfigRepository from litellm.types.roi_calculator import ( DEFAULT_PROMPT, + ROIBranchSpend, ROICompletionRequest, ROIIdentityMapResponse, ROIIdentityMapUpdate, @@ -40,6 +55,7 @@ from litellm.types.roi_calculator import ( ROISpendRecord, ROISummaryResponse, ROISyncStatus, + normalize_source_login, ) router: Final = APIRouter() @@ -52,6 +68,9 @@ _ROI_TAGS: Final[list[str | Enum]] = ["roi calculator"] # mutable-ok: FastAPI r class _StoredSettings(BaseModel): model_config = ConfigDict(extra="ignore") + source_provider: Literal["github", "gitlab"] = "github" + gitlab_api_url: str = "https://gitlab.com/api/v4" + gitlab_token: str = "" github_api_url: str = "https://api.github.com" github_token: str = "" estimator_key: str = "" @@ -181,6 +200,11 @@ async def _load_settings(repository: ConfigRepository) -> ROISettings: token: Final = decrypt_value_helper(stored.github_token, _SETTINGS_KEY) if stored.github_token else "" try: return ROISettings( + source_provider=stored.source_provider, + gitlab_api_url=stored.gitlab_api_url, + gitlab_token=SecretStr(decrypt_value_helper(stored.gitlab_token, _SETTINGS_KEY) or "") + if stored.gitlab_token + else SecretStr(""), github_api_url=stored.github_api_url, github_token=SecretStr(token or ""), estimator_key=SecretStr(decrypt_value_helper(stored.estimator_key, _SETTINGS_KEY) or "") @@ -202,8 +226,12 @@ async def _save_settings( settings: ROISettings, encrypted_token: str, encrypted_estimator_key: str, + encrypted_gitlab_token: str = "", ) -> None: stored: Final = _StoredSettings( + source_provider=settings.source_provider, + gitlab_api_url=settings.gitlab_api_url, + gitlab_token=encrypted_gitlab_token, github_api_url=settings.github_api_url, github_token=encrypted_token, estimator_key=encrypted_estimator_key, @@ -217,19 +245,28 @@ async def _save_settings( await repository.set_param(_SETTINGS_KEY, stored.model_dump(mode="json")) -async def _load_report(repository: ConfigRepository) -> ROIReport | None: +async def _load_report(repository: ConfigRepository, settings: ROISettings) -> ROIReport | None: parameter: Final = await repository.get_param(_REPORT_KEY) - if parameter is None: + if parameter is None or parameter.param_value is None: return None try: - return TypeAdapter(ROIReport).validate_python(parameter.param_value) + report: Final = TypeAdapter(ROIReport).validate_python(parameter.param_value) except ValidationError: raise HTTPException(status_code=500, detail="Stored ROI Calculator report is invalid.") from None + if ( + report.get("source_provider", "github") != settings.source_provider + or report.get("source_api_url", settings.github_api_url) != settings.source_api_url + ): + return None + return report def _public_settings(settings: ROISettings) -> ROISettingsResponse: models: Final = _router_models() return ROISettingsResponse( + source_provider=settings.source_provider, + gitlab_api_url=settings.gitlab_api_url, + has_gitlab_token=bool(settings.gitlab_token.get_secret_value()), github_api_url=settings.github_api_url, repos=settings.repos, estimator_model=settings.estimator_model, @@ -267,7 +304,14 @@ def _gateway_http_client() -> AsyncHTTPHandler: @lru_cache(maxsize=1) def _gateway_transport(app: FastAPI) -> httpx.ASGITransport: - return httpx.ASGITransport(app=app) + async def estimator_request(scope: Scope, receive: Receive, send: Send) -> None: + await app( + {**scope, "state": {**scope.get("state", {}), "litellm_roi_estimator": True}}, + receive, + send, + ) + + return httpx.ASGITransport(app=estimator_request) def _completion_caller(settings: ROISettings) -> CompletionCaller: @@ -317,6 +361,21 @@ def _spend_reader(repository: ConfigRepository) -> SpendReader: return get_spend +def _branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> BranchSpendReader: + async def get_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]: + return await read_branch_spend( + cast( # cast-ok: PrismaWrapper delegates methods dynamically + BranchSpendDatabase, repository.prisma_client.db + ), + start, + end, + repos, + casefold_repo=settings.source_provider == "github", + ) + + return get_spend + + @router.get( "/roi-calculator/settings", response_model=ROISettingsResponse, @@ -343,8 +402,26 @@ async def update_roi_calculator_settings( current: Final = await _load_settings(repository) if "github_api_url" in patch.model_fields_set and patch.github_api_url is None: raise HTTPException(status_code=422, detail="GitHub API URL cannot be null.") + if "gitlab_api_url" in patch.model_fields_set and patch.gitlab_api_url is None: + raise HTTPException(status_code=422, detail="GitLab API URL cannot be null.") + provider: Final = patch.source_provider or current.source_provider + gitlab_url: Final = patch.gitlab_api_url if patch.gitlab_api_url is not None else current.gitlab_api_url + gitlab_changed: Final = gitlab_url.rstrip("/") != current.gitlab_api_url.rstrip("/") + gitlab_token: Final = ( + (patch.gitlab_token or "") + if "gitlab_token" in patch.model_fields_set + else "" + if gitlab_changed + else current.gitlab_token.get_secret_value() + ) + encrypted_gitlab: Final = ( + TypeAdapter(str).validate_python(encrypt_value_helper(gitlab_token)) if gitlab_token else "" + ) github_api_url: Final = patch.github_api_url if patch.github_api_url is not None else current.github_api_url github_url_changed: Final = github_api_url.rstrip("/") != current.github_api_url.rstrip("/") + source_changed: Final = provider != current.source_provider or ( + gitlab_changed if provider == "gitlab" else github_url_changed + ) token_was_supplied: Final = "github_token" in patch.model_fields_set plaintext_token, encrypted_token = ( ( @@ -368,23 +445,28 @@ async def update_roi_calculator_settings( ) try: settings: Final = ROISettings( + source_provider=provider, + gitlab_api_url=gitlab_url, + gitlab_token=SecretStr(gitlab_token), github_api_url=github_api_url, github_token=SecretStr(plaintext_token), estimator_key=SecretStr(estimator_key), update_interval_minutes=patch.update_interval_minutes if patch.update_interval_minutes is not None else current.update_interval_minutes, - repos=patch.repos if patch.repos is not None else current.repos, + repos=patch.repos if patch.repos is not None else () if source_changed else current.repos, estimator_model=(patch.estimator_model if patch.estimator_model is not None else current.estimator_model), estimator_prompt=( patch.estimator_prompt if patch.estimator_prompt is not None else current.estimator_prompt ), backfill_days=(patch.backfill_days if patch.backfill_days is not None else current.backfill_days), - identity_map=current.identity_map, + identity_map=MappingProxyType({}) if source_changed else current.identity_map, ) except ValidationError as exc: raise HTTPException(status_code=422, detail=exc.errors(include_context=False)) from None - await _save_settings(repository, settings, encrypted_token, encrypted_estimator_key) + await _save_settings(repository, settings, encrypted_token, encrypted_estimator_key, encrypted_gitlab) + if source_changed: + await repository.set_param(_REPORT_KEY, None) return _public_settings(settings) @@ -400,7 +482,7 @@ async def get_roi_calculator_repositories( query: Annotated[str, Query(max_length=200)] = "", page: Annotated[int, Query(ge=1, le=1000)] = 1, ) -> ROIRepositoriesResponse: - github: Final = GitHub(await _load_settings(repository), transport) + github: Final = create_source(await _load_settings(repository), transport) try: repos, has_more = await github.repositories(query, page) except SourceError as exc: @@ -428,7 +510,7 @@ async def get_roi_calculator_sync_status( ) -> ROISyncStatus: status: Final = await SyncStore(repository.prisma_client).status() or manager.status settings: Final = await _load_settings(repository) - report: Final = await _load_report(repository) + report: Final = await _load_report(repository, settings) next_update: Final = _next_update(settings, status, report) return status.model_copy(update=MappingProxyType({"next_update": next_update.isoformat() if next_update else None})) @@ -448,7 +530,7 @@ async def start_roi_calculator_sync( settings: Final = await _load_settings(repository) public: Final = _public_settings(settings) if not public.ready: - raise HTTPException(status_code=409, detail="Connect GitHub, select repositories, and choose a router model.") + raise HTTPException(status_code=409, detail="Connect a source, select repositories, and choose a router model.") if not await manager.start( settings, repository, @@ -457,6 +539,7 @@ async def start_roi_calculator_sync( transport, _router_estimator_models(settings.estimator_model), SyncStore(repository.prisma_client), + branch_spend_reader=_branch_spend_reader(repository, settings), ): raise HTTPException(status_code=409, detail="A sync is already running.") return manager.status @@ -493,10 +576,10 @@ async def get_roi_calculator_report( sample: Final = summarize(sample_report(datetime.now(timezone.utc)), MappingProxyType({})) return ROIReportResponse(report=ROISummaryResponse.model_validate(sample)) - report: Final = await _load_report(repository) + settings: Final = await _load_settings(repository) + report: Final = await _load_report(repository, settings) if report is None: return ROIReportResponse(report=None) - settings: Final = await _load_settings(repository) summary: Final = summarize(report, settings.identity_map) return ROIReportResponse(report=ROISummaryResponse.model_validate(summary)) @@ -514,15 +597,22 @@ async def update_roi_calculator_identity_map( login: Final = update.github_login.strip().casefold() current: Final = await _load_settings(repository) current_stored: Final = await _load_stored_settings(repository) + try: + normalize_source_login(login, current.source_provider) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from None new_email: Final = normalize_email(update.email) if not login or (update.email is not None and not new_email): - raise HTTPException(status_code=422, detail="Enter a GitHub login and a valid email address.") + raise HTTPException(status_code=422, detail="Enter a source-control username and a valid email address.") identity_map: Final[Mapping[str, str]] = ( MappingProxyType({key: value for key, value in current.identity_map.items() if key != login}) if update.email is None else MappingProxyType({**current.identity_map, login: new_email}) ) settings: Final = ROISettings( + source_provider=current.source_provider, + gitlab_api_url=current.gitlab_api_url, + gitlab_token=current.gitlab_token, github_api_url=current.github_api_url, github_token=current.github_token, estimator_key=current.estimator_key, @@ -533,8 +623,10 @@ async def update_roi_calculator_identity_map( backfill_days=current.backfill_days, identity_map=identity_map, ) - await _save_settings(repository, settings, current_stored.github_token, current_stored.estimator_key) - report: Final = await _load_report(repository) + await _save_settings( + repository, settings, current_stored.github_token, current_stored.estimator_key, current_stored.gitlab_token + ) + report: Final = await _load_report(repository, settings) summary: Final = summarize(report, settings.identity_map) if report is not None else None return ROIIdentityMapResponse( report=ROISummaryResponse.model_validate(summary) if summary is not None else None, @@ -581,7 +673,7 @@ async def run_scheduled_sync() -> None: return store: Final = SyncStore(prisma_client) status: Final = await store.status() or _SYNC_MANAGER.status - report: Final = await _load_report(repository) + report: Final = await _load_report(repository, settings) next_update: Final = _next_update(settings, status, report) if next_update is None or next_update > datetime.now(timezone.utc): return @@ -593,6 +685,7 @@ async def run_scheduled_sync() -> None: estimator_models=_router_estimator_models(settings.estimator_model), coordinator=store, scheduled_interval=settings.update_interval_minutes, + branch_spend_reader=_branch_spend_reader(repository, settings), ) @@ -607,7 +700,7 @@ async def test_roi_calculator_connections( if not public.ready: raise HTTPException(status_code=409, detail="Choose repositories and an available estimator model first.") await _test_estimator_access(settings) - github: Final = GitHub(settings, transport) + github: Final = create_source(settings, transport) try: await github.test_repositories(settings.repos) except SourceError as exc: @@ -643,7 +736,7 @@ async def reset_roi_calculator_setup( current: Final = await _load_settings(repository) stored: Final = await _load_stored_settings(repository) settings: Final = current.model_copy(update=MappingProxyType({"repos": ()})) - await _save_settings(repository, settings, stored.github_token, stored.estimator_key) + await _save_settings(repository, settings, stored.github_token, stored.estimator_key, stored.gitlab_token) await store.clear_report() return _public_settings(settings) finally: diff --git a/litellm/proxy/roi_calculator/analytics.py b/litellm/proxy/roi_calculator/analytics.py index cb3ef46e5a4..7bf469f8936 100644 --- a/litellm/proxy/roi_calculator/analytics.py +++ b/litellm/proxy/roi_calculator/analytics.py @@ -3,6 +3,8 @@ from collections.abc import Mapping from typing import Final from litellm.types.roi_calculator import ( + ROIBranchAttribution, + ROIBranchMetrics, ROIPersonSummary, ROIPullRecord, ROIPullSummary, @@ -53,8 +55,12 @@ def _pull_summary( address: str, method: str, observed: frozenset[str], + branch_cost: ROIBranchAttribution, ) -> ROIPullSummary: return ROIPullSummary( + source_repo=pull.get("source_repo", ""), + source_branch=pull.get("source_branch", ""), + branch_cost=branch_cost, repo=pull["repo"], number=pull["number"], title=pull["title"], @@ -119,6 +125,9 @@ def _summarize_person( def summarize(report: ROIReport, mappings: Mapping[str, str]) -> ROISummary: + from litellm.proxy.roi_calculator.branch_spend import attribute_branches + + branch_costs: Final = attribute_branches(report["pulls"], report.get("branch_spend")) complete_scope: Final = not report.get("unavailable_repos", ()) observed: Final = frozenset( normalized for normalized in (normalize_email(row["email"]) for row in report["spend"]) if normalized @@ -143,7 +152,8 @@ def summarize(report: ROIReport, mappings: Mapping[str, str]) -> ROISummary: for key in sorted(people_keys) ) pull_summaries: Final = tuple( - _pull_summary(pull, address, method, observed) for pull, address, method in matched_pulls + _pull_summary(pull, address, method, observed, branch_costs[(pull["repo"], pull["number"])]) + for pull, address, method in matched_pulls ) eligible_emails: Final = frozenset(person["email"] for person in people if person["eligible"]) dates: Final = tuple( @@ -197,7 +207,26 @@ def summarize(report: ROIReport, mappings: Mapping[str, str]) -> ROISummary: ) summary_people: Final = tuple(sorted(people, key=lambda person: (-person["hours"], person["id"]))) summary_pulls: Final = tuple(sorted(pull_summaries, key=lambda pull: pull["merged_at"], reverse=True)) + branch_cohort: Final = tuple( + pull + for pull in pull_summaries + if pull["branch_cost"].status == "matched" and pull["estimate"]["status"] == "estimated" + ) + branch_spend: Final = sum(pull["branch_cost"].spend or 0 for pull in branch_cohort) + branch_hours: Final = sum(pull["estimate"]["hours"] or 0 for pull in branch_cohort) + linked: Final = frozenset((pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in branch_cohort) + unlinked: Final = tuple(row for row in report.get("branch_spend", ()) if (row.repo, row.branch) not in linked) return ROISummary( + source_provider=report.get("source_provider", "github"), + branch_metrics=ROIBranchMetrics( + spend=branch_spend, + hours=branch_hours, + cost_per_hour=branch_spend / branch_hours if complete_scope and branch_hours else None, + matched_pulls=sum(pull["branch_cost"].status == "matched" for pull in pull_summaries), + total_tagged_spend=sum(row.spend for row in report.get("branch_spend", ())), + unlinked_spend=sum(row.spend for row in unlinked), + ), + unlinked_branches=unlinked, id=report.get("id"), mode=report["mode"], start=report["start"], diff --git a/litellm/proxy/roi_calculator/branch_spend.py b/litellm/proxy/roi_calculator/branch_spend.py new file mode 100644 index 00000000000..f441683e843 --- /dev/null +++ b/litellm/proxy/roi_calculator/branch_spend.py @@ -0,0 +1,81 @@ +import json +from collections import Counter +from collections.abc import Mapping +from datetime import date, datetime, time, timedelta, timezone +from typing import Final, Protocol + +from pydantic import TypeAdapter + +from litellm.types.roi_calculator import ROIBranchAttribution, ROIBranchSpend, ROIPullRecord + + +class BranchSpendDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> object: ... + + +async def read_branch_spend( + database: BranchSpendDatabase, start: date, end: date, repos: tuple[str, ...], *, casefold_repo: bool = False +) -> tuple[ROIBranchSpend, ...]: + if not repos: + return () + query: Final = """ + WITH tagged AS ( + SELECT logs.spend, tags.repos[1] AS repo, tags.branches[1] AS branch + FROM "LiteLLM_SpendLogs" AS logs + CROSS JOIN LATERAL ( + SELECT array_agg(DISTINCT substring(tag FROM 6)) + FILTER (WHERE starts_with(tag, 'repo:')) AS repos, + array_agg(DISTINCT substring(tag FROM 8)) + FILTER (WHERE starts_with(tag, 'branch:')) AS branches + FROM jsonb_array_elements_text( + CASE WHEN jsonb_typeof(logs.request_tags) = 'array' + THEN logs.request_tags ELSE '[]'::jsonb END + ) AS tag + ) AS tags + WHERE logs."startTime" >= $1::text::timestamp AND logs."startTime" < $2::text::timestamp + AND cardinality(tags.repos) = 1 AND cardinality(tags.branches) = 1 + AND CASE logs.metadata -> 'litellm_roi_estimator' + WHEN 'true'::jsonb THEN false + WHEN 'false'::jsonb THEN true + ELSE NOT coalesce(logs.request_tags ? 'litellm-roi-estimator', false) + END + ) + SELECT CASE WHEN $4 THEN lower(repo) ELSE repo END AS repo, + branch, sum(spend)::double precision AS spend, count(*)::integer AS requests + FROM tagged + WHERE branch <> '' AND (CASE WHEN $4 THEN lower(repo) ELSE repo END) + IN (SELECT jsonb_array_elements_text($3::jsonb)) + GROUP BY 1, 2 + ORDER BY 1, 2 + """ + result: Final = await database.query_raw( + query, + datetime.combine(start, time.min, timezone.utc).isoformat(), + datetime.combine(end + timedelta(days=1), time.min, timezone.utc).isoformat(), + json.dumps(repos), + casefold_repo, + ) + return TypeAdapter(tuple[ROIBranchSpend, ...]).validate_python(result) + + +def attribute_branches( + pulls: tuple[ROIPullRecord, ...], spend: tuple[ROIBranchSpend, ...] | None +) -> Mapping[tuple[str, int], ROIBranchAttribution]: + counts: Final = Counter((pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in pulls) + costs: Final = {(row.repo, row.branch): row for row in spend or ()} + + def attribute(pull: ROIPullRecord) -> ROIBranchAttribution: + repo: Final = pull.get("source_repo", "") + branch: Final = pull.get("source_branch", "") + cost: Final = costs.get((repo, branch)) + if spend is None: + return ROIBranchAttribution(repo=repo, branch=branch, status="unavailable") + if not repo or not branch or cost is None: + return ROIBranchAttribution(repo=repo, branch=branch) + if counts[(repo, branch)] != 1: + return ROIBranchAttribution(repo=repo, branch=branch, status="ambiguous") + return ROIBranchAttribution( + repo=repo, branch=branch, spend=cost.spend, requests=cost.requests, status="matched" + ) + + return {(pull["repo"], pull["number"]): attribute(pull) for pull in pulls} diff --git a/litellm/proxy/roi_calculator/estimator.py b/litellm/proxy/roi_calculator/estimator.py index 4cb211f9cb0..0636d0b669c 100644 --- a/litellm/proxy/roi_calculator/estimator.py +++ b/litellm/proxy/roi_calculator/estimator.py @@ -112,14 +112,16 @@ class Estimator: missing_metadata_estimate: Final[ROIEstimate] = { "status": "needs_review", "hours": None, - "reasoning": ("GitHub did not provide all file or commit metadata. It was not sent for estimation."), + "reasoning": ( + "The repository source did not provide all file or commit metadata. It was not sent for estimation." + ), } return missing_metadata_estimate if len(evidence) > MAX_EVIDENCE_CHARS: oversized_evidence_estimate: Final[ROIEstimate] = { "status": "needs_review", "hours": None, - "reasoning": ("This PR exceeds the estimator's input limit. It was not truncated or scored."), + "reasoning": ("This change exceeds the estimator's input limit. It was not truncated or scored."), } return oversized_evidence_estimate system_message: Final[ROICompletionMessage] = { @@ -131,7 +133,6 @@ class Estimator: response_format: Final[ROIResponseFormat] = {"type": "json_object"} metadata: Final[ROICompletionMetadata] = { "tags": ("litellm-roi-estimator",), - "litellm_roi_estimator": True, } request: Final = ROICompletionRequest( model=self.settings.estimator_model, diff --git a/litellm/proxy/roi_calculator/github.py b/litellm/proxy/roi_calculator/github.py index f5134b84336..9f8aa8c26ae 100644 --- a/litellm/proxy/roi_calculator/github.py +++ b/litellm/proxy/roi_calculator/github.py @@ -31,8 +31,14 @@ class _GitHubUser(_GitHubModel): login: str | None = None +class _GitHubHeadRepository(_GitHubModel): + full_name: str = "" + + class _GitHubHead(_GitHubModel): sha: str = "" + ref: str = "" + repo: _GitHubHeadRepository | None = None class GitHubPullListItem(_GitHubModel): @@ -315,6 +321,7 @@ class GitHub: ) -> None: if client is not None and transport is not None: raise ValueError("Pass either an injected GitHub client or a transport.") + self._settings: Final = settings self._profiles: Mapping[str, str | None] = MappingProxyType({}) token: Final = settings.github_token.get_secret_value() self._headers: Final[Mapping[str, str]] = ( @@ -472,7 +479,11 @@ class GitHub: if address ) changed_files: Final = detail.changed_files if detail.changed_files is not None else len(files) + from litellm.proxy.roi_calculator.source import repository_tag + evidence: Final[ROIPullEvidence] = { + "source_repo": repository_tag(self._settings, detail.head.repo.full_name) if detail.head.repo else "", + "source_branch": detail.head.ref, "repo": repo, "number": detail.number, "title": detail.title, diff --git a/litellm/proxy/roi_calculator/gitlab.py b/litellm/proxy/roi_calculator/gitlab.py new file mode 100644 index 00000000000..2df35a0060b --- /dev/null +++ b/litellm/proxy/roi_calculator/gitlab.py @@ -0,0 +1,267 @@ +import asyncio +from collections.abc import Mapping +from datetime import date +from types import MappingProxyType +from typing import Final, TypeVar +from urllib.parse import quote + +import httpx +from pydantic import BaseModel, TypeAdapter + +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params +) +from litellm.proxy.roi_calculator.analytics import normalize_email +from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError +from litellm.proxy.roi_calculator.source import repository_tag +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.roi_calculator import ROIPullCommit, ROIPullEvidence, ROIPullFile, ROISettings + +_T: Final = TypeVar("_T", bound=BaseModel) + + +class _User(BaseModel): + username: str + public_email: str | None = None + + +class _Project(BaseModel): + id: int + path_with_namespace: str + visibility: str = "private" + archived: bool = False + + +class _MergeRequest(BaseModel): + iid: int + title: str + description: str | None = None + web_url: str + author: _User + merged_at: str | None + updated_at: str + sha: str | None = None + source_branch: str + source_project_id: int | None + changes_count: str | None = None + + def pull(self, source: _Project | None) -> GitHubPullListItem: + return GitHubPullListItem.model_validate( + { + "number": self.iid, + "title": self.title, + "body": self.description or "", + "html_url": self.web_url, + "user": {"login": self.author.username}, + "merged_at": self.merged_at, + "updated_at": self.updated_at, + "head": { + "sha": self.sha or "", + "ref": self.source_branch, + "repo": {"full_name": source.path_with_namespace} if source else None, + }, + } + ) + + +class _Diff(BaseModel): + new_path: str + old_path: str + diff: str = "" + new_file: bool = False + deleted_file: bool = False + renamed_file: bool = False + collapsed: bool = False + too_large: bool = False + + def file(self) -> ROIPullFile: + return ROIPullFile( + filename=self.new_path, + status="added" + if self.new_file + else "removed" + if self.deleted_file + else "renamed" + if self.renamed_file + else "modified", + additions=sum(line.startswith("+") for line in self.diff.splitlines()), + deletions=sum(line.startswith("-") for line in self.diff.splitlines()), + ) + + +class _Commit(BaseModel): + id: str + message: str + + +class GitLab: + def __init__(self, settings: ROISettings, transport: httpx.AsyncBaseTransport | None = None) -> None: + self.settings: Final = settings + token: Final = settings.gitlab_token.get_secret_value() + self.headers: Final = {"Accept": "application/json", **({"PRIVATE-TOKEN": token} if token else {})} + self.client: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.ROICalculator, + params={"timeout": 45, "follow_redirects": False, "transport": transport}, + ).client + self.close_client: Final = transport is not None + self.profiles: Mapping[str, str] = MappingProxyType({}) + self.projects: Mapping[int, _Project] = MappingProxyType({}) + self.source_project_slots: Final = asyncio.Semaphore(8) + + async def close(self) -> None: + if self.close_client: + await self.client.aclose() + + async def _request( + self, path: str, params: Mapping[str, str | int] | None = None, attempt: int = 0 + ) -> httpx.Response: + try: + response: Final = await self.client.get( + self.settings.gitlab_api_url + "/" + path, params=params, headers=self.headers + ) + except httpx.RequestError: + raise SourceError("Could not reach GitLab. Check the API URL and network connection.") from None + if response.status_code in (429, 502, 503, 504) and attempt < 2: + await asyncio.sleep(0.5 * (attempt + 1)) + return await self._request(path, params, attempt + 1) + if response.status_code != 200: + raise SourceError( + f"GitLab could not read this resource (HTTP {response.status_code}). " + "Check the project, token read_api scope, and project membership." + ) + return response + + async def _page( + self, path: str, model: type[_T], params: Mapping[str, str | int], page: int + ) -> tuple[tuple[_T, ...], bool]: + response: Final = await self._request(path, {**params, "per_page": 100, "page": page}) + try: + values: Final = TypeAdapter(tuple[object, ...]).validate_python(response.json()) + items: Final = tuple(model.model_validate(value) for value in values) + except ValueError: + raise SourceError("GitLab returned an invalid page of results.") from None + has_more: Final = response.headers.get("x-next-page", "") != "" or 'rel="next"' in response.headers.get( + "link", "" + ) + return items, has_more + + async def _all(self, path: str, model: type[_T], params: Mapping[str, str | int] | None = None) -> tuple[_T, ...]: + async def collect(page: int, previous: tuple[_T, ...]) -> tuple[_T, ...]: + items, more = await self._page(path, model, params or {}, page) + if not more: + return previous + items + if page >= 100: + raise SourceError("GitLab's pagination limit was reached. Narrow the reporting window.") + return await collect(page + 1, previous + items) + + return await collect(1, ()) + + async def _project(self, project: str | int) -> _Project: + if isinstance(project, int) and project in self.projects: + return self.projects[project] + response: Final = await self._request("projects/" + quote(str(project), safe="")) + try: + result: Final = _Project.model_validate(response.json()) + except ValueError: + raise SourceError("GitLab returned invalid project details.") from None + self.projects = MappingProxyType({**self.projects, result.id: result}) + return result + + async def repositories(self, query: str = "", page: int = 1) -> tuple[tuple[tuple[str, str, bool], ...], bool]: + params: Final = { + "simple": "true", + "search": query, + **({"membership": "true"} if self.headers.get("PRIVATE-TOKEN") else {}), + } + items, more = await self._page("projects", _Project, params, page) + return tuple((item.path_with_namespace, item.visibility, item.archived) for item in items), more + + async def test_repositories(self, repos: tuple[str, ...]) -> None: + async def test(repo: str) -> None: + project: Final = await self._project(repo) + await self._request(f"projects/{project.id}/merge_requests", {"state": "merged", "per_page": 1}) + + for repo in repos: + await test(repo) + + async def pulls(self, repo: str, start: date, end: date) -> tuple[GitHubPullListItem, ...]: + project: Final = await self._project(repo) + items: Final = await self._all( + f"projects/{project.id}/merge_requests", + _MergeRequest, + { + "state": "merged", + "scope": "all", + "updated_after": start.isoformat() + "T00:00:00Z", + "order_by": "updated_at", + "sort": "desc", + }, + ) + merged: Final = tuple( + item for item in items if item.merged_at and start.isoformat() <= item.merged_at[:10] <= end.isoformat() + ) + source_ids: Final = tuple(frozenset(item.source_project_id for item in merged)) + projects: Final = await asyncio.gather(*(self._source_project(source_id) for source_id in source_ids)) + sources: Final = MappingProxyType(dict(zip(source_ids, projects, strict=True))) + return tuple(item.pull(sources[item.source_project_id]) for item in merged) + + async def profile_email(self, login: str, *, fallback: str = "") -> str: + if login.casefold() in self.profiles: + return self.profiles[login.casefold()] + try: + users: Final = await self._all("users", _User, {"username": login}) + except SourceError: + return fallback + email: Final = next( + (normalize_email(user.public_email) for user in users if user.username.casefold() == login.casefold()), "" + ) + self.profiles = MappingProxyType({**self.profiles, login.casefold(): email}) + return email + + async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence: + project: Final = await self._project(repo) + path: Final = f"projects/{project.id}/merge_requests/{pull.number}" + response: Final = await self._request(path) + try: + detail: Final = _MergeRequest.model_validate(response.json()) + except ValueError: + raise SourceError("GitLab returned invalid merge request details.") from None + diffs: Final = await self._all(path + "/diffs", _Diff) + commits: Final = await self._all(path + "/commits", _Commit) + profile: Final = await self.profile_email(detail.author.username) + source: Final = await self._source_project(detail.source_project_id) + files: Final = tuple(diff.file() for diff in diffs) + return ROIPullEvidence( + repo=repo, + number=detail.iid, + title=detail.title, + body=detail.description or "", + url=detail.web_url, + login=detail.author.username, + emails=(profile,) if profile else (), + profile_email=profile, + commit_emails=(), + merged_at=detail.merged_at or "", + head_sha=detail.sha or "", + source_repo=repository_tag(self.settings, source.path_with_namespace) if source else "", + source_branch=detail.source_branch, + additions=sum(file["additions"] or 0 for file in files), + deletions=sum(file["deletions"] or 0 for file in files), + changed_files=len(files), + files=files, + commits=tuple(ROIPullCommit(sha=commit.id, message=commit.message) for commit in commits), + commit_count=len(commits), + incomplete_metadata=any(diff.collapsed or diff.too_large for diff in diffs) + or detail.changes_count is None + or not detail.changes_count.isdigit() + or int(detail.changes_count) != len(files), + ) + + async def _source_project(self, project_id: int | None) -> _Project | None: + if project_id is None: + return None + try: + async with self.source_project_slots: + return await self._project(project_id) + except SourceError: + return None diff --git a/litellm/proxy/roi_calculator/pull_cache.py b/litellm/proxy/roi_calculator/pull_cache.py index e1800fd0620..73c3688af1a 100644 --- a/litellm/proxy/roi_calculator/pull_cache.py +++ b/litellm/proxy/roi_calculator/pull_cache.py @@ -18,12 +18,15 @@ def cache_key( return None value: Final = json.dumps( ( - "pull-v1", - settings.github_api_url.rstrip("/"), + "pull-v2-branches", + settings.source_provider, + settings.source_api_url.rstrip("/"), context, - repo.casefold(), + repo.casefold() if settings.source_provider == "github" else repo, pull.number, head, + pull.head.ref if pull.head is not None else "", + pull.head.repo.full_name if pull.head is not None and pull.head.repo is not None else "", pull.title, pull.body or "", login.casefold(), @@ -36,7 +39,8 @@ def cache_key( def settings_fingerprint(settings: ROISettings) -> str: value: Final = json.dumps( ( - settings.github_api_url.rstrip("/"), + settings.source_provider, + settings.source_api_url.rstrip("/"), settings.repos, settings.estimator_model, settings.estimator_prompt, diff --git a/litellm/proxy/roi_calculator/sample.py b/litellm/proxy/roi_calculator/sample.py index fe5fbbaa866..1a3f65988d1 100644 --- a/litellm/proxy/roi_calculator/sample.py +++ b/litellm/proxy/roi_calculator/sample.py @@ -1,7 +1,14 @@ from datetime import datetime, timedelta from typing import Final -from litellm.types.roi_calculator import DEFAULT_PROMPT, ROIEstimate, ROIPullRecord, ROIReport, ROISpendRecord +from litellm.types.roi_calculator import ( + DEFAULT_PROMPT, + ROIBranchSpend, + ROIEstimate, + ROIPullRecord, + ROIReport, + ROISpendRecord, +) def sample_report(now: datetime) -> ROIReport: @@ -9,8 +16,10 @@ def sample_report(now: datetime) -> ROIReport: examples: Final = ( ("alex", "alex@example.com", "Add usage breakdown by model", 6.5, 18.2), ("jordan", "jordan@example.com", "Fix streaming response cancellation", 4.0, 12.8), - ("casey", "", "Add integration tests for billing", 5.5, 0.0), + ("casey", "", "Add integration tests for billing", 5.5, 7.4), ) + branches: Final = ("feature/model-usage", "fix/stream-cancellation", "test/billing-integration") + branch_costs: Final = (9.1, 6.4, 7.4) def pull(index: int, login: str, email: str, title: str, hours: float) -> ROIPullRecord: estimate: Final[ROIEstimate] = { @@ -23,6 +32,8 @@ def sample_report(now: datetime) -> ROIReport: "cached": False, } return ROIPullRecord( + source_repo="github.com/example/gateway", + source_branch=branches[index], repo="example/gateway", number=142 + index, title=title, @@ -47,9 +58,13 @@ def sample_report(now: datetime) -> ROIReport: spend: Final = tuple( ROISpendRecord(date=pulls[index]["merged_at"][:10], user_id=login, email=email, spend=cost, requests=150) for index, (login, email, _, _, cost) in enumerate(examples) - if email ) return ROIReport( + branch_spend=tuple( + ROIBranchSpend(repo="github.com/example/gateway", branch=branch, spend=cost, requests=75) + for branch, cost in zip(branches, branch_costs) + ) + + (ROIBranchSpend(repo="github.com/example/gateway", branch="feature/cost-export", spend=3.6, requests=30),), mode="demo", start=start.isoformat(), end=now.date().isoformat(), diff --git a/litellm/proxy/roi_calculator/source.py b/litellm/proxy/roi_calculator/source.py new file mode 100644 index 00000000000..40e1651a680 --- /dev/null +++ b/litellm/proxy/roi_calculator/source.py @@ -0,0 +1,33 @@ +from datetime import date +from typing import Final, Protocol +from urllib.parse import urlsplit + +import httpx + +from litellm.proxy.roi_calculator.github import GitHub, GitHubPullListItem +from litellm.types.roi_calculator import ROIPullEvidence, ROISettings + + +class RepositorySource(Protocol): + async def repositories(self, query: str = "", page: int = 1) -> tuple[tuple[tuple[str, str, bool], ...], bool]: ... + async def test_repositories(self, repos: tuple[str, ...]) -> None: ... + async def pulls(self, repo: str, start: date, end: date) -> tuple[GitHubPullListItem, ...]: ... + async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence: ... + async def profile_email(self, login: str, *, fallback: str = "") -> str: ... + async def close(self) -> None: ... + + +def repository_tag(settings: ROISettings, repo: str) -> str: + parsed: Final = urlsplit(settings.source_api_url) + host: Final = "github.com" if parsed.netloc == "api.github.com" else parsed.netloc + prefix: Final = parsed.path.removesuffix("/api/v4").removesuffix("/api/v3").rstrip("/") + value: Final = host + prefix + "/" + repo + return value.casefold() if settings.source_provider == "github" else value + + +def create_source(settings: ROISettings, transport: httpx.AsyncBaseTransport | None = None) -> RepositorySource: + if settings.source_provider == "gitlab": + from litellm.proxy.roi_calculator.gitlab import GitLab + + return GitLab(settings, transport) + return GitHub(settings, transport) diff --git a/litellm/proxy/roi_calculator/sync.py b/litellm/proxy/roi_calculator/sync.py index 65a2cb38a17..e134c23dab1 100644 --- a/litellm/proxy/roi_calculator/sync.py +++ b/litellm/proxy/roi_calculator/sync.py @@ -11,11 +11,14 @@ import httpx from pydantic import BaseModel, ConfigDict, Field, TypeAdapter from typing_extensions import ReadOnly, TypedDict, Unpack +from litellm._logging import verbose_proxy_logger from litellm.proxy.roi_calculator.estimator import CompletionCaller, Estimator, EstimatorModel, cache_context -from litellm.proxy.roi_calculator.github import GitHub, GitHubPullListItem, SourceError +from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError from litellm.proxy.roi_calculator.pull_cache import cache_key, settings_fingerprint +from litellm.proxy.roi_calculator.source import RepositorySource, create_source, repository_tag from litellm.repositories.chunked_in import find_many_in from litellm.types.roi_calculator import ( + ROIBranchSpend, ROIEstimate, ROIPullEvidence, ROIPullRecord, @@ -31,6 +34,10 @@ _REPORT_ADAPTER: Final = TypeAdapter(ROIReport) _JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object]) +class _BranchSpendFields(TypedDict, total=False): + branch_spend: ReadOnly[tuple[ROIBranchSpend, ...]] + + class _ConfigParam(Protocol): @property def param_value(self) -> object: ... @@ -164,7 +171,7 @@ class GitHubFactory(Protocol): self, settings: ROISettings, transport: httpx.AsyncBaseTransport | None, - ) -> GitHub: ... + ) -> RepositorySource: ... class SpendReader(Protocol): @@ -175,6 +182,10 @@ class SpendReader(Protocol): ) -> Awaitable[tuple[ROISpendRecord, ...]]: ... +class BranchSpendReader(Protocol): + def __call__(self, start: date, end: date, repos: tuple[str, ...]) -> Awaitable[tuple[ROIBranchSpend, ...]]: ... + + class SyncClock(Protocol): def __call__(self) -> datetime: ... @@ -210,7 +221,9 @@ async def _estimate_with_fallback( return estimate -async def _unavailable_record(github: GitHub, repo: str, pull: GitHubPullListItem, error: SourceError) -> ROIPullRecord: +async def _unavailable_record( + github: RepositorySource, settings: ROISettings, repo: str, pull: GitHubPullListItem, error: SourceError +) -> ROIPullRecord: login: Final = pull.user.login if pull.user and pull.user.login else "deleted-user" profile: Final = await github.profile_email(login) estimate: Final[ROIEstimate] = { @@ -219,6 +232,10 @@ async def _unavailable_record(github: GitHub, repo: str, pull: GitHubPullListIte "reasoning": f"PR metadata could not be read: {error} Run analysis again to retry this PR.", } return ROIPullRecord( + source_repo=repository_tag(settings, pull.head.repo.full_name) + if pull.head and pull.head.repo and pull.head.repo.full_name + else "", + source_branch=pull.head.ref if pull.head else "", repo=repo, number=pull.number, title=pull.title, @@ -258,25 +275,27 @@ class _RepositoryBatch(NamedTuple): stage: str -async def _read_repository(github: GitHub, repo: str, start: date, end: date) -> _RepositoryPulls: +async def _read_repository(github: RepositorySource, repo: str, start: date, end: date) -> _RepositoryPulls: try: return _RepositoryPulls(repo, await github.pulls(repo, start, end)) except SourceError: return _RepositoryPulls(repo, (), unavailable=True) -async def _read_repositories(github: GitHub, repos: tuple[str, ...], start: date, end: date) -> _RepositoryBatch: +async def _read_repositories( + github: RepositorySource, repos: tuple[str, ...], start: date, end: date +) -> _RepositoryBatch: groups: Final = await asyncio.gather(*(_read_repository(github, repo, start, end) for repo in repos)) unavailable: Final = tuple(group.repo for group in groups if group.unavailable) if len(unavailable) == len(repos): raise SourceError( - "GitHub could not read any selected repository. No new report was published; " + "The repository source could not read any selected repository. No new report was published; " "check repository access or try analysis again later." ) queue: Final = tuple(chain.from_iterable(((group.repo, pull) for pull in group.pulls) for group in groups)) if unavailable and not queue: raise SourceError( - f"GitHub could not read {', '.join(unavailable)}, and the accessible repositories returned no pull requests. " + f"The repository source could not read {', '.join(unavailable)}, and the accessible repositories returned no merged changes. " "No new report was published; check repository access or try analysis again later." ) warnings: Final = ( @@ -301,13 +320,13 @@ async def _read_repositories(github: GitHub, repos: tuple[str, ...], start: date def _processed_records(processed: tuple[_ProcessedPull, ...]) -> Mapping[int, ROIPullRecord]: if processed and all(item.metadata_unavailable for item in processed): raise SourceError( - "GitHub could not provide PR metadata. No new report was published; try analysis again later." + "The repository source could not provide PR metadata. No new report was published; try analysis again later." ) if any(item.record["estimate"]["status"] == "error" for item in processed) and not any( item.record["estimate"]["status"] == "estimated" for item in processed ): raise SourceError( - "The estimator could not score any pull requests. No new report was published; " + "The estimator could not score any merged changes. No new report was published; " "check the estimator connection or try analysis again later." ) return MappingProxyType({item.position: item.record for item in processed}) @@ -332,7 +351,7 @@ async def _cache_estimated_pull( class SyncManager: def __init__( self, - github_factory: GitHubFactory = GitHub, + github_factory: GitHubFactory = create_source, clock: SyncClock = _utc_now, ) -> None: self._github_factory: Final = github_factory @@ -379,6 +398,7 @@ class SyncManager: estimator_models: tuple[EstimatorModel, ...] | None = None, coordinator: SyncCoordinator | None = None, scheduled_interval: float = 0, + branch_spend_reader: BranchSpendReader | None = None, ) -> bool: async with self._start_lock: if not settings.repos or not settings.estimator_model: @@ -410,7 +430,15 @@ class SyncManager: self._owner = owner self._task = asyncio.create_task( self._run( - settings, repository, spend_reader, complete, github_transport, estimator_models, coordinator, owner + settings, + repository, + spend_reader, + complete, + github_transport, + estimator_models, + coordinator, + owner, + branch_spend_reader, ) ) return True @@ -452,6 +480,7 @@ class SyncManager: estimator_models: tuple[EstimatorModel, ...] | None, coordinator: SyncCoordinator | None, owner: str, + branch_spend_reader: BranchSpendReader | None, ) -> None: monitor: Final = asyncio.create_task(self._heartbeat(asyncio.current_task(), coordinator, owner)) github: Final = self._github_factory(settings, github_transport) @@ -477,7 +506,7 @@ class SyncManager: ) self._update_status( phase="estimates", - stage="Estimating new or changed pull requests", + stage="Estimating merged changes", total=len(queue), ) estimator: Final = Estimator(settings, complete, estimator_models) @@ -521,7 +550,7 @@ class SyncManager: try: evidence: Final = await github.evidence(repo, pull) except SourceError as exc: - unavailable: Final = await _unavailable_record(github, repo, pull, exc) + unavailable: Final = await _unavailable_record(github, settings, repo, pull, exc) self._update_estimate_progress(unavailable["estimate"]) return _ProcessedPull(index, unavailable, metadata_unavailable=True) estimate: Final = await _estimate_with_fallback(estimator, evidence) @@ -531,7 +560,13 @@ class SyncManager: "number": evidence["number"], "title": evidence["title"], "body": evidence["body"], - "head": MappingProxyType({"sha": evidence["head_sha"]}), + "head": MappingProxyType( + { + "sha": evidence["head_sha"], + "ref": evidence.get("source_branch", ""), + "repo": pull.head.repo if pull.head is not None else None, + } + ), "user": MappingProxyType({"login": evidence["login"]}), "merged_at": evidence["merged_at"], "updated_at": evidence["merged_at"], @@ -559,7 +594,26 @@ class SyncManager: worker_task.cancel() await asyncio.gather(*workers, return_exceptions=True) processed_by_index: Final = _processed_records(processed) + records: Final = tuple(processed_by_index[index] for index in range(len(queue))) + branch_repos: Final = tuple( + sorted( + frozenset( + ( + *(repository_tag(settings, repo) for repo in settings.repos), + *(pull.get("source_repo", "") for pull in records), + ) + ) + - {""} + ) + ) + branch_spend: Final = await branch_spend_reader(start, end, branch_repos) if branch_spend_reader else None + branch_fields: Final[_BranchSpendFields] = ( + {"branch_spend": branch_spend} if branch_spend is not None else {} + ) report: Final = ROIReport( + source_provider=settings.source_provider, + source_api_url=settings.source_api_url, + **branch_fields, mode="live", start=start.isoformat(), end=end.isoformat(), @@ -569,7 +623,7 @@ class SyncManager: estimator_prompt=settings.estimator_prompt, effort_basis="without_ai", spend=spend, - pulls=tuple(processed_by_index[index] for index in range(len(queue))), + pulls=records, settings_fingerprint=settings_fingerprint(settings), warnings=repositories.warnings, unavailable_repos=repositories.unavailable_repos, @@ -605,6 +659,7 @@ class SyncManager: except SourceError as exc: self._update_status(phase="error", stage="Sync failed", error=str(exc)) except Exception: # noqa: BLE001 - background job boundary records a safe failure for every source error + verbose_proxy_logger.exception("ROI Calculator sync failed") self._update_status( phase="error", stage="Sync failed", @@ -646,6 +701,8 @@ class SyncManager: def _cached_record(self, pull: ROIPullRecord) -> ROIPullRecord: estimate: Final = _ESTIMATE_ADAPTER.validate_python(MappingProxyType({**pull["estimate"], "cached": True})) return ROIPullRecord( + source_repo=pull.get("source_repo", ""), + source_branch=pull.get("source_branch", ""), repo=pull["repo"], number=pull["number"], title=pull["title"], @@ -672,6 +729,8 @@ class SyncManager: key: str | None, ) -> ROIPullRecord: return ROIPullRecord( + source_repo=evidence.get("source_repo", ""), + source_branch=evidence.get("source_branch", ""), repo=evidence["repo"], number=evidence["number"], title=evidence["title"], diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94a0f424a09..b71b834a31c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -158,6 +158,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "autorouter_savings_estimate", "autorouter_baseline_observation", "used_client_oauth_token", + "litellm_roi_estimator", ) ) @@ -211,6 +212,7 @@ def _get_spend_logs_metadata( usage_object=None, guardrail_information=None, internal_call_origin=None, + litellm_roi_estimator=False, eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, @@ -244,6 +246,7 @@ def _get_spend_logs_metadata( router_metadata=router_metadata, azure_spillover=azure_spillover, used_client_oauth_token=used_client_oauth_token, + litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, ) _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") diff --git a/litellm/types/roi_calculator.py b/litellm/types/roi_calculator.py index a15bcbdac9b..b1c15d9a197 100644 --- a/litellm/types/roi_calculator.py +++ b/litellm/types/roi_calculator.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final, Literal -from pydantic import BaseModel, ConfigDict, Field, SecretStr, StrictFloat, StrictInt, field_validator +from pydantic import BaseModel, ConfigDict, Field, SecretStr, StrictFloat, StrictInt, ValidationInfo, field_validator from typing_extensions import NotRequired, ReadOnly, TypedDict DEFAULT_PROMPT: Final = ( @@ -11,18 +11,22 @@ DEFAULT_PROMPT: Final = ( ) -def _normalize_login(value: str) -> str: +def normalize_source_login(value: str, provider: str = "github") -> str: import re login: Final = value.strip().casefold() - if re.fullmatch(r"[A-Za-z0-9_\[\]-]+", login) is None: - raise ValueError("Enter a valid GitHub username.") + pattern: Final = r"[A-Za-z0-9_.-]+" if provider == "gitlab" else r"[A-Za-z0-9_\[\]-]+" + if re.fullmatch(pattern, login) is None: + raise ValueError("Enter a valid source-control username.") return login class ROISettings(BaseModel): model_config = ConfigDict(frozen=True) + source_provider: Literal["github", "gitlab"] = "github" + gitlab_api_url: str = "https://gitlab.com/api/v4" + gitlab_token: SecretStr = SecretStr("") github_api_url: str = "https://api.github.com" github_token: SecretStr = SecretStr("") estimator_key: SecretStr = SecretStr("") @@ -33,6 +37,10 @@ class ROISettings(BaseModel): update_interval_minutes: float = Field(default=1440, ge=0, le=43200, allow_inf_nan=False) identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({})) + @property + def source_api_url(self) -> str: + return self.gitlab_api_url if self.source_provider == "gitlab" else self.github_api_url + @field_validator("update_interval_minutes") @classmethod def validate_update_interval(cls, value: float) -> float: @@ -40,14 +48,14 @@ class ROISettings(BaseModel): raise ValueError("Choose manual updates (0), or an interval of at least 5 minutes.") return value - @field_validator("github_api_url") + @field_validator("github_api_url", "gitlab_api_url") @classmethod def normalize_github_api_url(cls, value: str) -> str: from urllib.parse import urlsplit normalized: Final[str] = value.strip().rstrip("/") if not normalized: - raise ValueError("A GitHub API URL is required.") + raise ValueError("A source API URL is required.") parsed: Final = urlsplit(normalized) if ( parsed.scheme != "https" @@ -57,26 +65,30 @@ class ROISettings(BaseModel): or parsed.query or parsed.fragment ): - raise ValueError("Use an HTTPS GitHub API URL without credentials, query, or fragment.") + raise ValueError("Use an HTTPS source API URL without credentials, query, or fragment.") return normalized @field_validator("repos") @classmethod - def validate_repositories(cls, values: tuple[str, ...]) -> tuple[str, ...]: + def validate_repositories(cls, values: tuple[str, ...], info: ValidationInfo) -> tuple[str, ...]: import re normalized_values: Final = tuple(repo.strip().rstrip("/").removesuffix(".git") for repo in values) normalized: Final = tuple( repo for index, repo in enumerate(normalized_values) if repo not in normalized_values[:index] ) + pattern: Final = ( + r"[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+" + if info.data.get("source_provider") == "gitlab" + else r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+" + ) invalid_repositories: Final = tuple( repo for repo in normalized - if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo) is None - or any(part in (".", "..") for part in repo.split("/")) + if re.fullmatch(pattern, repo) is None or any(part in (".", "..") for part in repo.split("/")) ) if invalid_repositories: - raise ValueError("Repositories must use owner/repo format.") + raise ValueError("Use owner/repo format, or group/subgroup/project for GitLab.") return normalized @field_validator("estimator_prompt") @@ -89,24 +101,29 @@ class ROISettings(BaseModel): @field_validator("identity_map") @classmethod - def normalize_identity_map(cls, values: Mapping[str, str]) -> Mapping[str, str]: + def normalize_identity_map(cls, values: Mapping[str, str], info: ValidationInfo) -> Mapping[str, str]: from litellm.proxy.roi_calculator.analytics import normalize_email normalized: Final[Mapping[str, str]] = MappingProxyType( { - _normalize_login(login): normalize_email(address) + normalize_source_login( + login, "gitlab" if info.data.get("source_provider") == "gitlab" else "github" + ): normalize_email(address) for login, address in values.items() if normalize_email(address) } ) if len(normalized) != len(values): - raise ValueError("Each identity needs a GitHub username and a valid gateway email.") + raise ValueError("Each identity needs a source-control username and a valid gateway email.") return normalized class ROISettingsUpdate(BaseModel): model_config = ConfigDict(extra="forbid") + source_provider: Literal["github", "gitlab"] | None = None + gitlab_api_url: str | None = None + gitlab_token: str | None = None github_api_url: str | None = None github_token: str | None = None estimator_key: str | None = None @@ -118,6 +135,9 @@ class ROISettingsUpdate(BaseModel): class ROISettingsResponse(BaseModel): + source_provider: Literal["github", "gitlab"] = "github" + gitlab_api_url: str = "https://gitlab.com/api/v4" + has_gitlab_token: bool = False github_api_url: str repos: tuple[str, ...] estimator_model: str @@ -180,6 +200,8 @@ class ROIEstimate(TypedDict): class ROIPullRecord(TypedDict): + source_repo: NotRequired[ReadOnly[str]] + source_branch: NotRequired[ReadOnly[str]] repo: ReadOnly[str] number: ReadOnly[int] title: ReadOnly[str] @@ -199,7 +221,34 @@ class ROIPullRecord(TypedDict): cache_key: ReadOnly[str | None] +class ROIBranchSpend(BaseModel): + repo: str + branch: str + spend: float + requests: int + + +class ROIBranchAttribution(BaseModel): + repo: str + branch: str + spend: float | None = None + requests: int = 0 + status: Literal["matched", "unattributed", "ambiguous", "unavailable"] = "unattributed" + + +class ROIBranchMetrics(BaseModel): + spend: float = 0 + hours: float = 0 + cost_per_hour: float | None = None + matched_pulls: int = 0 + total_tagged_spend: float = 0 + unlinked_spend: float = 0 + + class ROIReport(TypedDict): + source_api_url: NotRequired[ReadOnly[str]] + source_provider: NotRequired[ReadOnly[Literal["github", "gitlab"]]] + branch_spend: NotRequired[ReadOnly[tuple[ROIBranchSpend, ...]]] mode: ReadOnly[str] start: ReadOnly[str] end: ReadOnly[str] @@ -232,6 +281,8 @@ class ROIPullCommit(TypedDict): class ROIPullEvidence(TypedDict): + source_repo: NotRequired[ReadOnly[str]] + source_branch: NotRequired[ReadOnly[str]] repo: ReadOnly[str] number: ReadOnly[int] title: ReadOnly[str] @@ -273,6 +324,9 @@ class ROIPersonSummary(TypedDict): class ROIPullSummary(TypedDict): + branch_cost: ReadOnly[ROIBranchAttribution] + source_repo: NotRequired[ReadOnly[str]] + source_branch: NotRequired[ReadOnly[str]] repo: ReadOnly[str] number: ReadOnly[int] title: ReadOnly[str] @@ -318,6 +372,9 @@ class ROITrendDay(TypedDict): class ROISummary(TypedDict): + source_provider: ReadOnly[Literal["github", "gitlab"]] + branch_metrics: ReadOnly[ROIBranchMetrics] + unlinked_branches: ReadOnly[tuple[ROIBranchSpend, ...]] id: ReadOnly[str | None] mode: ReadOnly[str] start: ReadOnly[str] @@ -375,6 +432,9 @@ class ROIEstimateResponse(BaseModel): class ROIPullResponse(BaseModel): + source_repo: str = "" + source_branch: str = "" + branch_cost: ROIBranchAttribution = Field(default_factory=lambda: ROIBranchAttribution(repo="", branch="")) repo: str number: int title: str @@ -404,6 +464,9 @@ class ROITrendResponse(BaseModel): class ROISummaryResponse(BaseModel): + source_provider: Literal["github", "gitlab"] = "github" + branch_metrics: ROIBranchMetrics = Field(default_factory=ROIBranchMetrics) + unlinked_branches: tuple[ROIBranchSpend, ...] = () id: str | None mode: str start: str @@ -431,7 +494,7 @@ class ROIIdentityMapUpdate(BaseModel): @field_validator("github_login") @classmethod def normalize_login(cls, value: str) -> str: - return _normalize_login(value) + return value.strip().casefold() class ROIIdentityMapResponse(BaseModel): @@ -478,7 +541,6 @@ class ROICompletionMessage(TypedDict): class ROICompletionMetadata(TypedDict): tags: ReadOnly[tuple[str, ...]] - litellm_roi_estimator: ReadOnly[bool] class ROIResponseFormat(TypedDict): diff --git a/tests/integration/spend/test_roi_branch_spend.py b/tests/integration/spend/test_roi_branch_spend.py new file mode 100644 index 00000000000..c90aa0073cd --- /dev/null +++ b/tests/integration/spend/test_roi_branch_spend.py @@ -0,0 +1,79 @@ +import json +import os +import uuid +from datetime import date +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from prisma import Prisma +from psycopg import sql + +from litellm.proxy.roi_calculator.branch_spend import read_branch_spend + + +@pytest.mark.asyncio +async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() -> None: + schema: Final = f"integration_roi_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + parsed: Final = urlsplit(url) + scoped: Final = urlunsplit(parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}))) + repo: Final = "gitlab.com/group/project" + tags: Final = (f"repo:{repo}", "branch:feature/one") + rows: Final = ( + ("2026-09-01 00:00:00", 2, tags), + ("2026-09-30 23:59:59.999", 3, tags + tags), + ("2026-10-01 00:00:00", 100, tags), + ("2026-08-31 23:59:59.999", 100, tags), + ("2026-09-15 00:00:00", 100, tags + ("branch:conflict",)), + ("2026-09-15 00:00:00", 100, tags + ("repo:gitlab.com/other/project",)), + ("2026-09-15 00:00:00", 100, ("branch:feature/one",)), + ("2026-09-15 00:00:00", 11, tags + ("litellm-roi-estimator",)), + ("2026-09-15 00:00:00", 0, (f"repo:{repo}", "branch:free")), + ("2026-09-15 00:00:00", 7, (f"repo:{repo}", "branch:Feature/one")), + ) + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute( + sql.SQL( + 'CREATE TABLE {}."LiteLLM_SpendLogs" ' + '("startTime" timestamp, spend float, request_tags jsonb, metadata jsonb)' + ).format(sql.Identifier(schema)) + ) + for timestamp, spend, request_tags in rows: + setup.execute( + sql.SQL( + 'INSERT INTO {}."LiteLLM_SpendLogs" ("startTime", spend, request_tags) ' + 'VALUES (%s::timestamp, %s, %s::jsonb)' + ).format(sql.Identifier(schema)), + (timestamp, spend, json.dumps(request_tags)), + ) + for marker, spend, extra_tags in ( + (True, 100, ()), + (True, 100, ("litellm-roi-estimator",)), + (False, 13, ("litellm-roi-estimator",)), + (None, 100, ("litellm-roi-estimator",)), + ): + setup.execute( + sql.SQL('INSERT INTO {}."LiteLLM_SpendLogs" VALUES (%s::timestamp, %s, %s::jsonb, %s::jsonb)').format( + sql.Identifier(schema) + ), + ( + "2026-09-15 00:00:00", + spend, + json.dumps(tags + extra_tags), + json.dumps({"litellm_roi_estimator": marker}), + ), + ) + database: Final = Prisma(datasource={"url": scoped}) + await database.connect() + try: + result: Final = await read_branch_spend(database, date(2026, 9, 1), date(2026, 9, 30), (repo,)) + finally: + await database.disconnect() + costs: Final = {row.branch: (row.spend, row.requests) for row in result} + assert costs == {"feature/one": (18, 3), "Feature/one": (7, 1), "free": (0, 1)} + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) diff --git a/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py b/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py index 66b9df69996..938c813f363 100644 --- a/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py @@ -2,20 +2,25 @@ import asyncio import json from collections.abc import Mapping from datetime import datetime, timezone +from math import isclose from types import MappingProxyType from typing import Final, cast +import httpx import pytest from apscheduler.schedulers.asyncio import AsyncIOScheduler -from fastapi import FastAPI +from fastapi import FastAPI, Request from fastapi.testclient import TestClient from pydantic import TypeAdapter from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.management_endpoints.roi_calculator_endpoints import ( _estimator_models_from_deployments, + _gateway_transport, _next_update, + get_github_transport, get_roi_config_repository, register_scheduled_sync, router, @@ -23,11 +28,58 @@ from litellm.proxy.management_endpoints.roi_calculator_endpoints import ( ) from litellm.proxy.roi_calculator.estimator import estimator_options from litellm.proxy.roi_calculator.sample import sample_report -from litellm.types.roi_calculator import ROIReport, ROISettings, ROISyncStatus +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.roi_calculator import ROIReport, ROISettings, ROISummaryResponse, ROISyncStatus _JSON_HEADERS: Final = MappingProxyType({"content-type": "application/json"}) +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/v1/chat/completions", "/v1/responses", "/v1/messages")) +@pytest.mark.parametrize("string_metadata", (False, True)) +async def test_only_internal_estimator_transport_can_mark_persisted_spend(path: str, string_metadata: bool) -> None: + from litellm.proxy.proxy_server import ProxyConfig + + app: Final = FastAPI() + tags: Final = ("repo:org/repo", "branch:feature", "litellm-roi-estimator") + forged: Final = {"tags": tags, "litellm_roi_estimator": True} + metadata: Final = json.dumps(forged) if string_metadata else forged + body: Final = {"model": "test-model", "metadata": metadata, "litellm_metadata": metadata} + now: Final = datetime(2026, 9, 15, tzinfo=timezone.utc) + + @app.post(path) + async def log_request(request: Request) -> Mapping[str, object]: + data: Final = await add_litellm_data_to_request( + data=await request.json(), + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", metadata={"litellm_roi_estimator": True}), + proxy_config=ProxyConfig(), + ) + payload: Final = get_logging_payload( + kwargs={"model": "test-model", "response_cost": 0.25, "litellm_params": data}, + response_obj={"id": "test-request", "usage": {"prompt_tokens": 10, "completion_tokens": 5}}, + start_time=now, + end_time=now, + ) + return { + "metadata": json.loads(payload["metadata"]), + "tags": json.loads(payload["request_tags"]), + "spend": payload["spend"], + } + + async with ( + httpx.AsyncClient(transport=_gateway_transport(app), base_url="http://test") as internal, + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as external, + ): + for client, expected in ((external, False), (internal, True), (external, False)): + response: Final = await client.post(path, json=body, headers={"x-litellm-roi-estimator": "true"}) + assert response.status_code == 200 + logged: Final = response.json() + assert logged["metadata"].get("litellm_roi_estimator") is expected + assert set(logged["tags"]) == set(tags) + assert logged["spend"] == 0.25 + + @pytest.mark.asyncio async def test_repeated_startup_keeps_one_roi_schedule() -> None: scheduler: Final = AsyncIOScheduler() @@ -60,7 +112,7 @@ class _ConfigRepository: async def get_param(self, param_name: str) -> _Parameter | None: value: Final = self.values.get(param_name) - return _Parameter(value) if value is not None else None + return _Parameter(value) if param_name in self.values else None async def set_param(self, param_name: str, param_value: object) -> object: _assert_json_round_trip(param_value) @@ -68,11 +120,14 @@ class _ConfigRepository: return self.values[param_name] -def _client(role: LitellmUserRoles, repository: _ConfigRepository) -> TestClient: +def _client( + role: LitellmUserRoles, repository: _ConfigRepository, transport: httpx.AsyncBaseTransport | None = None +) -> TestClient: app: Final = FastAPI() app.include_router(router) app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role) app.dependency_overrides[get_roi_config_repository] = lambda: repository + app.dependency_overrides[get_github_transport] = lambda: transport return TestClient(app) @@ -161,6 +216,42 @@ def test_github_api_url_must_use_https() -> None: assert not repository.values +@pytest.mark.parametrize( + "patch", ({"github_api_url": None}, {"gitlab_api_url": None}, {"repos": ["invalid"]}, {"estimator_prompt": " "}) +) +def test_invalid_connection_settings_are_rejected_without_saving(patch: Mapping[str, object]) -> None: + repository: Final = _ConfigRepository() + client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository) + assert client.put("/roi-calculator/settings", json=patch).status_code == 422 + assert not repository.values + + +@pytest.mark.parametrize("upstream_status", (200, 403)) +def test_public_gitlab_repository_browser_and_errors(upstream_status: int) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v4/projects" + assert request.url.params["search"] == "gateway" + assert "PRIVATE-TOKEN" not in request.headers + return httpx.Response( + upstream_status, json=[{"id": 1, "path_with_namespace": "group/gateway"}], headers={"x-next-page": "2"} + ) + + repository: Final = _ConfigRepository() + client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository, httpx.MockTransport(respond)) + assert client.put("/roi-calculator/settings", json={"source_provider": "gitlab"}).status_code == 200 + response: Final = client.get("/roi-calculator/repositories", params={"query": "gateway"}) + if upstream_status == 200: + assert response.status_code == 200 + assert response.json() == { + "repositories": [{"name": "group/gateway", "visibility": "private", "archived": False}], + "page": 1, + "has_more": True, + } + else: + assert response.status_code == 502 + assert "HTTP 403" in response.json()["detail"] + + @pytest.mark.parametrize("role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) @pytest.mark.parametrize( "method,path,body", @@ -209,8 +300,16 @@ def test_sample_preview_does_not_change_live_settings_or_report() -> None: client: Final = _client(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, repository) response: Final = client.get("/roi-calculator/report", params={"mode": "demo"}) assert response.status_code == 200 - assert response.json()["report"]["mode"] == "demo" - assert response.json()["report"]["metrics"]["cost_per_hour"] > 0 + report: Final = ROISummaryResponse.model_validate(response.json()["report"]) + assert report.mode == "demo" + assert report.metrics.cost_per_hour is not None and report.metrics.cost_per_hour > 0 + assert all(pull.branch_cost.status == "matched" and (pull.branch_cost.spend or 0) > 0 for pull in report.pulls) + assert any(not pull.matched for pull in report.pulls) + assert isclose(report.branch_metrics.spend, sum(pull.branch_cost.spend or 0 for pull in report.pulls)) + assert report.branch_metrics.unlinked_spend > 0 + assert isclose( + report.branch_metrics.total_tagged_spend, report.branch_metrics.spend + report.branch_metrics.unlinked_spend + ) assert not repository.values assert client.get("/roi-calculator/report").json()["report"] is None @@ -264,3 +363,51 @@ def test_manual_match_recalculates_saved_report_and_removal_restores_cohort() -> assert removed.status_code == 200 assert not removed.json()["identity_map"] assert removed.json()["report"]["metrics"] == before.json()["report"]["metrics"] + + +def test_switching_sources_clears_report_and_identities_and_keeps_tokens_private( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "roi-calculator-test-salt-key-0123456789") + repository: Final = _ConfigRepository() + client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository) + saved: Final = client.put( + "/roi-calculator/settings", + json={"source_provider": "gitlab", "gitlab_token": "private-gitlab-test", "repos": ["group/subgroup/project"]}, + ) + assert saved.status_code == 200 + assert saved.json()["has_gitlab_token"] is True + assert "private-gitlab-test" not in saved.text + assert "private-gitlab-test" not in str(repository.values) + assert client.get("/roi-calculator/report").json()["report"] is None + matched: Final = client.put( + "/roi-calculator/identity-map", json={"github_login": "dev.name", "email": "dev@example.test"} + ) + assert matched.status_code == 200 + assert matched.json()["identity_map"] == {"dev.name": "dev@example.test"} + switched: Final = client.put("/roi-calculator/settings", json={"source_provider": "github"}) + assert switched.status_code == 200 + assert switched.json()["identity_map"] == {} + assert switched.json()["repos"] == [] + assert client.get("/roi-calculator/report").json()["report"] is None + changed_host: Final = client.put( + "/roi-calculator/settings", + json={"source_provider": "gitlab", "gitlab_api_url": "https://git.example.test/api/v4"}, + ) + assert changed_host.json()["has_gitlab_token"] is False + + +def test_old_source_report_is_not_returned_when_matching_new_source_identity() -> None: + repository: Final = _ConfigRepository() + client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository) + assert client.put("/roi-calculator/settings", json={"source_provider": "gitlab"}).status_code == 200 + old_report: Final = sample_report(datetime.now(timezone.utc)) + serialized: Final = TypeAdapter(dict[str, object]).validate_json(TypeAdapter(ROIReport).dump_json(old_report)) + asyncio.run(repository.set_param("roi_calculator_report", serialized)) + assert client.get("/roi-calculator/report").json()["report"] is None + matched: Final = client.put( + "/roi-calculator/identity-map", json={"github_login": "dev.name", "email": "dev@example.test"} + ) + assert matched.status_code == 200 + assert matched.json()["report"] is None + assert matched.json()["identity_map"] == {"dev.name": "dev@example.test"} diff --git a/tests/unit/proxy/roi_calculator/test_analytics.py b/tests/unit/proxy/roi_calculator/test_analytics.py index 2968c294b99..9dd4986c7e5 100644 --- a/tests/unit/proxy/roi_calculator/test_analytics.py +++ b/tests/unit/proxy/roi_calculator/test_analytics.py @@ -145,3 +145,45 @@ def test_email_normalization_rejects_private_or_unusable_addresses() -> None: assert normalize_email("123+alice@users.noreply.github.com") == "" assert normalize_email("alice") == "" assert normalize_email("") == "" + + +def test_branch_costs_are_independent_of_identity_and_never_count_reused_branches_twice() -> None: + from litellm.types.roi_calculator import ROIBranchSpend + + base: Final = _pull(emails=()) + pulls: Final[tuple[ROIPullRecord, ...]] = ( + {**base, "number": 1, "source_repo": "gitlab.com/group/repo", "source_branch": "feature"}, + {**base, "number": 2, "source_repo": "gitlab.com/group/repo", "source_branch": "reused"}, + {**base, "number": 3, "source_repo": "gitlab.com/group/repo", "source_branch": "reused"}, + {**base, "number": 4, "source_repo": "gitlab.com/group/repo", "source_branch": "missing"}, + {**base, "number": 5, "source_repo": "gitlab.com/group/repo", "source_branch": "free"}, + { + **_pull(emails=(), estimate_status="error", hours=None), + "number": 6, + "source_repo": "gitlab.com/group/repo", + "source_branch": "pending", + }, + ) + report: Final[ROIReport] = { + **_report(pulls), + "branch_spend": ( + ROIBranchSpend(repo="gitlab.com/group/repo", branch="feature", spend=12, requests=2), + ROIBranchSpend(repo="gitlab.com/group/repo", branch="reused", spend=7, requests=1), + ROIBranchSpend(repo="gitlab.com/group/repo", branch="free", spend=0, requests=1), + ROIBranchSpend(repo="gitlab.com/group/repo", branch="pending", spend=9, requests=1), + ), + } + result: Final = summarize(report, EMPTY_IDENTITY_MAP) + costs: Final = {pull["number"]: pull["branch_cost"] for pull in result["pulls"]} + assert costs[1].spend == 12 + assert costs[2].status == costs[3].status == "ambiguous" + assert costs[2].spend is None + assert costs[4].spend is None and costs[4].status == "unattributed" + assert costs[5].spend == 0 and costs[5].status == "matched" + assert result["branch_metrics"].cost_per_hour == 12 / 8 + assert result["branch_metrics"].unlinked_spend == 16 + assert result["branch_metrics"].matched_pulls == 3 + assert result["branch_metrics"].spend == 12 + assert result["metrics"]["matched_spend"] == 0 + incomplete: Final = summarize({**report, "unavailable_repos": ("other/repo",)}, EMPTY_IDENTITY_MAP) + assert incomplete["branch_metrics"].cost_per_hour is None diff --git a/tests/unit/proxy/roi_calculator/test_branch_spend.py b/tests/unit/proxy/roi_calculator/test_branch_spend.py new file mode 100644 index 00000000000..ac3f49ebc57 --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_branch_spend.py @@ -0,0 +1,32 @@ +import json +from datetime import date +from typing import Final + +import pytest + +from litellm.proxy.roi_calculator.branch_spend import read_branch_spend +from litellm.types.roi_calculator import ROIBranchSpend + + +class _SpendDatabase: + async def query_raw(self, query: str, *args: object) -> object: + assert args == ( + "2026-01-31T00:00:00+00:00", + "2026-02-01T00:00:00+00:00", + json.dumps(("gitlab.com/group/project",)), + False, + ) + return [{"repo": "gitlab.com/group/project", "branch": "feature", "spend": 0.000027, "requests": 3}] + + +@pytest.mark.asyncio +async def test_branch_spend_includes_the_final_utc_day_and_preserves_fractional_costs() -> None: + result: Final = await read_branch_spend( + _SpendDatabase(), date(2026, 1, 31), date(2026, 1, 31), ("gitlab.com/group/project",) + ) + assert result == (ROIBranchSpend(repo="gitlab.com/group/project", branch="feature", spend=0.000027, requests=3),) + + +@pytest.mark.asyncio +async def test_no_repositories_returns_no_spend_without_querying_the_database() -> None: + assert await read_branch_spend(_SpendDatabase(), date(2026, 1, 1), date(2026, 1, 31), ()) == () diff --git a/tests/unit/proxy/roi_calculator/test_gitlab.py b/tests/unit/proxy/roi_calculator/test_gitlab.py new file mode 100644 index 00000000000..260a1bd9b7e --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_gitlab.py @@ -0,0 +1,281 @@ +import asyncio +from datetime import date +from typing import Final + +import httpx +import pytest +from pydantic import SecretStr + +from litellm.proxy.roi_calculator.estimator import metadata_evidence +from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError +from litellm.proxy.roi_calculator.gitlab import GitLab +from litellm.types.roi_calculator import ROISettings + + +@pytest.mark.asyncio +async def test_fork_lookups_overlap_with_a_bounded_number_of_requests() -> None: + started: Final[asyncio.Queue[int]] = asyncio.Queue() + release: Final = tuple(asyncio.Event() for _ in range(9)) + source_ids: Final = (*range(2, 11), 3) + + async def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/projects/group/repo"): + return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"}) + if request.url.path.endswith("/merge_requests"): + return httpx.Response( + 200, + json=[ + { + "iid": index, + "title": "Fix parser", + "web_url": f"https://gitlab.com/group/repo/-/merge_requests/{index}", + "author": {"username": "dev"}, + "merged_at": "2026-09-30T12:00:00Z", + "updated_at": "2026-09-30T12:00:00Z", + "source_branch": f"fix/{index}", + "source_project_id": source_id, + } + for index, source_id in enumerate(source_ids) + ], + ) + project_id: Final = int(request.url.path.rsplit("/", 1)[1]) + started.put_nowait(project_id) + await release[project_id - 2].wait() + if project_id == 3: + return httpx.Response(404) + return httpx.Response(200, json={"id": project_id, "path_with_namespace": f"fork-{project_id}/repo"}) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + pending: Final = asyncio.create_task(source.pulls("group/repo", date(2026, 9, 1), date(2026, 9, 30))) + try: + first_wave: Final = tuple([await asyncio.wait_for(started.get(), timeout=1) for _ in range(8)]) + assert len(set(first_wave)) == 8 + assert started.empty() + release[first_wave[0] - 2].set() + next_id: Final = await asyncio.wait_for(started.get(), timeout=1) + assert next_id not in first_wave + for event in release: + event.set() + pulls: Final = await asyncio.wait_for(pending, timeout=1) + assert tuple(pull.head.repo.full_name if pull.head and pull.head.repo else None for pull in pulls) == tuple( + None if source_id == 3 else f"fork-{source_id}/repo" for source_id in source_ids + ) + assert started.empty() + finally: + for event in release: + event.set() + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + await source.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing_fork,source_id", ((False, 2), (True, 2), (False, None))) +async def test_gitlab_paginates_nested_projects_and_keeps_source_code_out_of_estimates( + missing_fork: bool, source_id: int | None +) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["PRIVATE-TOKEN"] == "test-only-token" + assert request.url.host == "git.example.test" + path: Final = request.url.path + detail: Final = { + "iid": 8, + "title": "Fix parser", + "description": "Handle empty input", + "web_url": "https://git.example.test/g/sub/p/-/merge_requests/8", + "author": {"username": "dev.name"}, + "merged_at": "2026-09-30T23:59:59Z", + "updated_at": "2026-10-01T00:00:00Z", + "sha": "sha", + "source_branch": "fix/parser", + "source_project_id": source_id, + "changes_count": "1", + } + if path.endswith("/projects/g/sub/p"): + assert "%2F" in str(request.url) + return httpx.Response(200, json={"id": 1, "path_with_namespace": "g/sub/p"}) + if path.endswith("/projects/2"): + return ( + httpx.Response(404) + if missing_fork + else httpx.Response(200, json={"id": 2, "path_with_namespace": "dev/fork"}) + ) + if path.endswith("/merge_requests"): + assert request.url.params["scope"] == "all" + if request.url.params["page"] == "1": + return httpx.Response( + 200, json=[{**detail, "iid": 7, "merged_at": "2026-10-01T00:00:00Z"}], headers={"x-next-page": "2"} + ) + return httpx.Response(200, json=[detail]) + if path.endswith("/merge_requests/8"): + return httpx.Response(200, json=detail) + if path.endswith("/diffs"): + return httpx.Response( + 200, + json=[ + { + "new_path": "parser.py", + "old_path": "parser.py", + "diff": "@@ -1 +1 @@\n---old-code\n+++private-code", + } + ], + ) + if path.endswith("/commits"): + return httpx.Response( + 200, json=[{"id": "sha", "message": "Fix empty input", "author_email": "untrusted@example.test"}] + ) + if path.endswith("/users"): + return httpx.Response(200, json=[{"username": "dev.name", "public_email": "dev@example.test"}]) + raise AssertionError(path) + + settings: Final = ROISettings( + source_provider="gitlab", + gitlab_api_url="https://git.example.test/api/v4", + gitlab_token=SecretStr("test-only-token"), + repos=("g/sub/p",), + ) + client: Final = GitLab(settings, httpx.MockTransport(respond)) + try: + pulls: Final = await client.pulls("g/sub/p", date(2026, 9, 1), date(2026, 9, 30)) + assert tuple(pull.number for pull in pulls) == (8,) + evidence: Final = await client.evidence("g/sub/p", pulls[0]) + assert evidence["source_repo"] == ("" if missing_fork or source_id is None else "git.example.test/dev/fork") + assert evidence["source_branch"] == "fix/parser" + assert evidence["emails"] == ("dev@example.test",) + assert evidence["commit_emails"] == () + assert (evidence["additions"], evidence["deletions"]) == (1, 1) + assert not evidence["incomplete_metadata"] + assert "private-code" not in metadata_evidence(evidence).model_dump_json() + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", (301, 401, 403, 404)) +async def test_gitlab_errors_do_not_follow_redirects_or_disclose_upstream_content(status: int) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.host == "gitlab.com" + return httpx.Response(status, text="secret-upstream-response", headers={"location": "https://untrusted.test/"}) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + try: + with pytest.raises(SourceError, match=f"HTTP {status}") as error: + await source.test_repositories(("group/project",)) + assert "secret-upstream-response" not in str(error.value) + finally: + await source.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token", ("", "test-token")) +async def test_gitlab_repository_browser_preserves_visibility_pagination_and_membership(token: str) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.params["search"] == "gateway" + assert request.url.params["page"] == "2" + assert (request.url.params.get("membership") == "true") == bool(token) + return httpx.Response( + 200, + json=[ + { + "id": 1, + "path_with_namespace": "group/sub/gateway", + "visibility": "internal", + "archived": True, + } + ], + headers={"link": '; rel="next"'}, + ) + + source: Final = GitLab( + ROISettings(source_provider="gitlab", gitlab_token=SecretStr(token)), httpx.MockTransport(respond) + ) + try: + assert await source.repositories("gateway", 2) == ((("group/sub/gateway", "internal", True),), True) + finally: + await source.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "resource,message", + ( + ("projects", "page of results"), + ("projects/group/repo", "project details"), + ("projects/1/merge_requests/8", "merge request details"), + ), +) +async def test_gitlab_rejects_malformed_responses(resource: str, message: str) -> None: + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/" + resource): + return httpx.Response(200, json={"private-error": "must not be disclosed"}) + return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"}) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + operation: Final = ( + source.repositories() + if resource == "projects" + else source.test_repositories(("group/repo",)) + if resource == "projects/group/repo" + else source.evidence("group/repo", GitHubPullListItem(number=8, title="Fix", updated_at="2026-09-30")) + ) + try: + with pytest.raises(SourceError, match=message): + await operation + finally: + await source.close() + + +@pytest.mark.asyncio +async def test_gitlab_connection_failure_is_sanitized_and_profile_uses_fallback() -> None: + def respond(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("private host detail", request=request) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + try: + with pytest.raises(SourceError, match="Could not reach GitLab") as error: + await source.repositories() + assert "private host detail" not in str(error.value) + assert await source.profile_email("alice", fallback="known@example.test") == "known@example.test" + finally: + await source.close() + + +@pytest.mark.asyncio +async def test_gitlab_stops_an_endless_pagination_response() -> None: + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/projects/group/repo"): + return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"}) + assert int(request.url.params["page"]) <= 100 + return httpx.Response(200, json=[], headers={"x-next-page": "101"}) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + try: + with pytest.raises(SourceError, match="pagination limit"): + await source.pulls("group/repo", date(2026, 9, 1), date(2026, 9, 30)) + finally: + await source.close() + + +@pytest.mark.asyncio +async def test_gitlab_retries_transient_errors_and_checks_merge_request_access() -> None: + statuses: Final = iter((429, 503, 200)) + reads: Final = iter(("/api/v4/projects/group/repo", "/api/v4/projects/1/merge_requests")) + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/projects/group/repo"): + status: Final = next(statuses) + if status != 200: + return httpx.Response(status) + assert request.url.path == next(reads) + return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"}) + assert request.url.path == next(reads) + assert request.url.params["state"] == "merged" + return httpx.Response(200, json=[]) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + try: + await source.test_repositories(("group/repo",)) + assert next(reads, None) is None + assert next(statuses, None) is None + finally: + await source.close() diff --git a/tests/unit/proxy/roi_calculator/test_sync.py b/tests/unit/proxy/roi_calculator/test_sync.py index f58bc396d94..a0749f230fa 100644 --- a/tests/unit/proxy/roi_calculator/test_sync.py +++ b/tests/unit/proxy/roi_calculator/test_sync.py @@ -14,6 +14,7 @@ from litellm.proxy.roi_calculator.estimator import CompletionCaller from litellm.proxy.roi_calculator.github import GitHubPullListItem from litellm.proxy.roi_calculator.sync import SpendReader, SyncManager, read_spend from litellm.types.roi_calculator import ( + ROIBranchSpend, ROICompletionRequest, ROIReport, ROISettings, @@ -28,7 +29,7 @@ _PULL_LIST_JSON: Final = """[ "body": "Preserve UTC behavior.", "merged_at": "2026-09-12T12:00:00Z", "updated_at": "2026-09-12T12:00:00Z", - "head": {"sha": "abcdef"}, + "head": {"sha": "abcdef", "ref": "feature", "repo": {"full_name": "org/repo"}}, "user": {"login": "alice"} } ]""" @@ -39,7 +40,7 @@ _PULL_DETAIL_JSON: Final = """{ "html_url": "https://github.com/org/repo/pull/42", "user": {"login": "alice"}, "merged_at": "2026-09-12T12:00:00Z", - "head": {"sha": "abcdef"}, + "head": {"sha": "abcdef", "ref": "feature", "repo": {"full_name": "org/repo"}}, "additions": 1, "deletions": 1, "changed_files": 1, @@ -242,10 +243,116 @@ async def test_unchanged_estimated_pull_refreshes_identity_without_model_call() assert manager.status.reused == 1 report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"]) assert report["pulls"][0]["estimate"].get("cached") is True + assert report["pulls"][0]["source_branch"] == "feature" + assert report["pulls"][0]["source_repo"] == "github.com/org/repo" assert report["pulls"][0]["profile_email"] == "new@example.com" assert report["pulls"][0]["emails"] == ("alice@example.com", "new@example.com") +def _gitlab_transport(source_path: str | None, *, details_fail: bool = False) -> httpx.MockTransport: + detail: Final = { + "iid": 42, + "title": "Fix timezone conversion", + "description": "Preserve UTC behavior.", + "web_url": "https://gitlab.com/org/repo/-/merge_requests/42", + "author": {"username": "alice"}, + "merged_at": "2026-09-12T12:00:00Z", + "updated_at": "2026-09-12T12:00:00Z", + "sha": "abcdef", + "source_branch": "feature", + "source_project_id": 2, + "changes_count": "1", + } + + def respond(request: httpx.Request) -> httpx.Response: + path: Final = request.url.path + if path.endswith("/projects/org/repo"): + return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/repo"}) + if path.endswith("/projects/2"): + return ( + httpx.Response(200, json={"id": 2, "path_with_namespace": source_path}) + if source_path + else httpx.Response(404) + ) + if path.endswith("/merge_requests"): + return httpx.Response( + 200, json=[detail, {**detail, "iid": 43, "source_branch": "other"}] if details_fail else [detail] + ) + if path.endswith("/merge_requests/43"): + return httpx.Response(200, json={**detail, "iid": 43, "source_branch": "other"}) + if path.endswith("/merge_requests/42"): + return httpx.Response(404) if details_fail else httpx.Response(200, json=detail) + if path.endswith("/diffs"): + return httpx.Response(200, json=[{"new_path": "time.py", "old_path": "time.py", "diff": "+fixed"}]) + if path.endswith("/commits"): + return httpx.Response(200, json=[{"id": "abcdef", "message": "Fix timezone conversion"}]) + if path.endswith("/users"): + return httpx.Response(200, json=[{"username": "alice", "public_email": "alice@example.com"}]) + raise AssertionError(path) + + return httpx.MockTransport(respond) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("before,after", [(None, "dev/fork"), ("dev/fork", None), ("dev/fork", "dev/renamed")]) +async def test_gitlab_cache_refreshes_branch_attribution_when_source_access_changes( + before: str | None, after: str | None +) -> None: + settings: Final = _settings().model_copy(update={"source_provider": "gitlab"}) + repository: Final = _ReportRepository() + manager: Final = SyncManager(clock=_fixed_now) + + async def branch_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]: + return (ROIBranchSpend(repo="gitlab.com/" + (after or "dev/fork"), branch="feature", spend=2.5, requests=3),) + + assert await manager.start(settings, repository, _spend_reader(), _completion(), _gitlab_transport(before)) + await _wait_until_finished(manager) + assert manager.status.phase == "complete" + assert await manager.start( + settings, repository, _spend_reader(), _completion(), _gitlab_transport(after), branch_spend_reader=branch_spend + ) + await _wait_until_finished(manager) + assert manager.status.phase == "complete" + report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"]) + assert report["pulls"][0]["source_repo"] == ("gitlab.com/" + after if after else "") + result: Final = summarize(report, {}) + assert result["pulls"][0]["branch_cost"].status == ("matched" if after else "unattributed") + + async def unexpected_completion(request: ROICompletionRequest) -> object: + raise AssertionError("Unchanged source metadata must reuse the estimate") + + assert await manager.start(settings, repository, _spend_reader(), unexpected_completion, _gitlab_transport(after)) + await _wait_until_finished(manager) + assert manager.status.phase == "complete" + assert manager.status.reused == 1 + + +@pytest.mark.asyncio +async def test_unreadable_gitlab_details_keep_known_branch_costs() -> None: + repository: Final = _ReportRepository() + manager: Final = SyncManager(clock=_fixed_now) + settings: Final = _settings().model_copy(update={"source_provider": "gitlab"}) + + async def branch_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]: + return (ROIBranchSpend(repo="gitlab.com/dev/fork", branch="feature", spend=2.5, requests=3),) + + assert await manager.start( + settings, + repository, + _spend_reader(), + _completion(), + _gitlab_transport("dev/fork", details_fail=True), + branch_spend_reader=branch_spend, + ) + await _wait_until_finished(manager) + report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"]) + result: Final = summarize(report, {}) + assert result["pulls"][0]["branch_cost"].spend == 2.5 + assert result["pulls"][0]["estimate"]["status"] == "needs_review" + assert result["branch_metrics"].matched_pulls == 1 + assert result["branch_metrics"].cost_per_hour is None + + @pytest.mark.asyncio async def test_read_spend_joins_user_emails_and_preserves_unmatched_identities() -> None: spend: Final = await read_spend( @@ -461,6 +568,8 @@ async def test_one_unreadable_pr_preserves_other_estimates_in_report() -> None: assert manager.status.phase == "complete" assert manager.status.estimated == 1 assert manager.status.needs_attention == 1 + assert report["pulls"][1]["source_repo"] == "github.com/org/repo" + assert report["pulls"][1]["source_branch"] == "feature" def _repository_outage_transport( diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 7a6b933d86e..a3de9328437 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -3530,6 +3530,24 @@ def test_get_spend_logs_metadata_keeps_user_agent(): assert _get_spend_logs_metadata(None)["user_agent"] is None +@pytest.mark.parametrize( + "metadata,expected", + ( + (None, False), + ({}, False), + ({"tags": ["litellm-roi-estimator"]}, False), + ({"litellm_roi_estimator": None}, False), + ({"litellm_roi_estimator": "true"}, False), + ({"litellm_roi_estimator": False}, False), + ({"litellm_roi_estimator": True}, True), + ), +) +def test_new_spend_logs_always_have_an_explicit_roi_estimator_marker( + metadata: dict[str, object] | None, expected: bool +) -> None: + assert _get_spend_logs_metadata(metadata)["litellm_roi_estimator"] is expected + + @pytest.mark.parametrize( "client_sent_oauth_token, custom_llm_provider, expected", [ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorDialogs.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorDialogs.tsx index 91626a41268..39d7ef582ef 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorDialogs.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorDialogs.tsx @@ -14,7 +14,7 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; -import { effortNote, estimateLabel } from "./roiCalculatorData"; +import { effortNote, estimateLabel, branchCostLabel } from "./roiCalculatorData"; import type { ROIIdentityMapUpdate, ROIPull, ROISummary } from "./roiCalculatorData"; import type { ROIPerson } from "./roiCalculatorData"; @@ -29,28 +29,37 @@ export function PullReasoningDialog({ summary: ROISummary | null; onClose: () => void; }) { + const titleRef = React.useRef(null); return ( !open && onClose()}> - + {pull && ( <> - {pull.title} + + {pull.title} + {pull.repo} #{pull.number} · {pull.login} -
-

Estimated engineering hours

-

{estimateLabel(pull.estimate)}

-

+

+
+
+
Estimated effort
+
{estimateLabel(pull.estimate)}
+
+
+
Recorded AI cost
+
{branchCostLabel(pull)}
+ {pull.branch_cost?.status === "matched" && ( +
{pull.branch_cost.requests} requests
+ )} +
+
+

{effortNote(pull.estimate.effort_basis ?? summary?.effort_basis)}

- {pull.estimate.evidence_source === "pr_metadata" && ( -

- Based on PR descriptions, file change counts, and commit metadata. -

- )}

Reasoning

@@ -66,6 +75,34 @@ export function PullReasoningDialog({
Email match
{pull.email || "Not matched"}
+
+

Track costs for this branch

+ {pull.branch_cost?.status === "matched" && ( +

+ {pull.branch_cost.spend?.toFixed(8)} USD across {pull.branch_cost.requests} requests +

+ )} + {pull.source_repo && pull.source_branch ? ( + <> +

Send both tags with each gateway request from this branch:

+ 
+                    {JSON.stringify(
+                      { metadata: { tags: [`repo:${pull.source_repo}`, `branch:${pull.source_branch}`] } },
+                      null,
+                      2,
+                    )}
+
+

+ Retained requests in the report’s UTC period. Branch names are case-sensitive. Reused branches + cannot be split between changes. +

+ + ) : ( +

+ The source repository or branch is unavailable. Sync again to refresh its metadata. +

+ )} +
{summary?.estimator_prompt && (
Estimator prompt @@ -80,7 +117,7 @@ export function PullReasoningDialog({ target="_blank" rel="noopener noreferrer" > - View on GitHub + View on {summary?.source_provider === "gitlab" ? "GitLab" : "GitHub"} )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx index 1501938e9c5..8cd3e47f296 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx @@ -1,6 +1,5 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { ReactNode } from "react"; import { apiClient } from "@/components/networking"; import ROICalculatorView from "./ROICalculatorView"; @@ -13,21 +12,6 @@ vi.mock("@/components/networking", () => ({ put: vi.fn(), }, })); -vi.mock("@/components/ui/chart", () => ({ - ChartContainer: ({ children }: { children: ReactNode }) =>
{children}
, - ChartLegend: () => null, - ChartLegendContent: () => null, - ChartTooltip: () => null, - ChartTooltipContent: () => null, -})); -vi.mock("recharts", () => ({ - Bar: () => null, - CartesianGrid: () => null, - ComposedChart: ({ children }: { children: ReactNode }) =>
{children}
, - Line: () => null, - XAxis: () => null, - YAxis: () => null, -})); const summary = { id: null, @@ -73,6 +57,15 @@ const summary = { pulls: [ { repo: "org/repo", + source_repo: "github.com/org/repo", + source_branch: "feature/routing", + branch_cost: { + repo: "github.com/org/repo", + branch: "feature/routing", + status: "matched", + spend: 8, + requests: 12, + }, number: 42, title: "Improve request routing", url: "https://github.com/org/repo/pull/42", @@ -131,6 +124,7 @@ const idleStatus = { describe("ROICalculatorView", () => { beforeEach(() => { + window.history.replaceState(null, "", "/roi-calculator/"); vi.mocked(apiClient.get).mockReset(); vi.mocked(apiClient.put).mockReset(); vi.mocked(apiClient.post).mockReset(); @@ -145,8 +139,8 @@ describe("ROICalculatorView", () => { it("shows the spend summary and opens an accessible pull reasoning dialog", async () => { render(); - expect(await screen.findByText("Spend per estimated engineering hour")).toBeInTheDocument(); - expect(screen.getByText("$3.00")).toBeInTheDocument(); + expect(await screen.findByText("Gateway AI cost")).toBeInTheDocument(); + expect(screen.getByText("$20.00")).toBeInTheDocument(); fireEvent.click(screen.getByRole("button", { name: "Open estimate for org/repo pull request 42" })); expect(await screen.findByRole("dialog")).toBeInTheDocument(); @@ -157,6 +151,19 @@ describe("ROICalculatorView", () => { ); }); + it("separates the overview, people, and branch reports into three tabs", async () => { + render(); + expect(await screen.findByRole("heading", { name: "Where AI costs are matched" })).toBeVisible(); + fireEvent.click(screen.getByRole("tab", { name: "Branches" })); + expect(screen.getByRole("heading", { name: "Costs by branch" })).toBeVisible(); + expect(screen.getByRole("cell", { name: "$8.00" })).toBeVisible(); + fireEvent.click(screen.getByRole("tab", { name: "People" })); + expect(screen.getByRole("heading", { name: "People and account matches" })).toBeVisible(); + fireEvent.click(screen.getByRole("tab", { name: "Overview" })); + expect(screen.getByRole("heading", { name: "Highest-cost changes" })).toBeVisible(); + expect(screen.queryByRole("radio")).not.toBeInTheDocument(); + }); + it("shows incomplete repository results without a spend-per-hour figure", async () => { const warning = "Incomplete report: could not read org/unavailable. Spend-per-hour figures are unavailable."; vi.mocked(apiClient.get).mockImplementation((path: string) => { @@ -179,7 +186,8 @@ describe("ROICalculatorView", () => { expect(await screen.findByRole("alert")).toHaveTextContent(warning); expect(screen.getByRole("button", { name: "Open estimate for org/repo pull request 42" })).toBeInTheDocument(); expect(screen.queryByText("$3.00")).not.toBeInTheDocument(); - fireEvent.click(screen.getByText("Calculation details")); + fireEvent.click(screen.getByRole("tab", { name: "People" })); + fireEvent.click(screen.getByText("How this is calculated")); expect( screen.getByText("Spend per estimated hour is unavailable until all selected repositories can be read."), ).toBeVisible(); @@ -201,7 +209,7 @@ describe("ROICalculatorView", () => { render(); - expect(await screen.findByText("Spend per estimated engineering hour")).toBeInTheDocument(); + expect(await screen.findByText("Gateway AI cost")).toBeInTheDocument(); expect(screen.getByRole("note")).toHaveTextContent("Read-only access"); expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Cancel sync" })).not.toBeInTheDocument(); @@ -210,8 +218,8 @@ describe("ROICalculatorView", () => { expect(screen.getByText("alice-work")).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "alice-work" })).not.toBeInTheDocument(); - fireEvent.click(screen.getByRole("tab", { name: "Settings" })); - expect(screen.getByLabelText("GitHub token")).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Settings" })); + expect(screen.getByLabelText("GitHub token (optional for public repositories)")).toBeDisabled(); expect(screen.queryByRole("button", { name: "Save settings" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); }); @@ -244,9 +252,119 @@ describe("ROICalculatorView", () => { render(); - expect(await screen.findByRole("heading", { name: "Connect GitHub to get started" })).toBeInTheDocument(); - expect(screen.getByLabelText("GitHub token")).toHaveAttribute("type", "password"); - expect(screen.getAllByText("Connect GitHub to get started")).toHaveLength(1); + expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeInTheDocument(); + expect(screen.getByLabelText("GitHub token (optional for public repositories)")).toHaveAttribute( + "type", + "password", + ); + expect(screen.getAllByText("Connect your repositories")).toHaveLength(1); + }); + + it.each(["github", "gitlab"])("only permits unauthenticated repository browsing for GitLab: %s", async (provider) => { + const publicSettings = { + ...settings, + source_provider: provider, + gitlab_api_url: "https://gitlab.com/api/v4", + has_github_token: false, + has_gitlab_token: false, + }; + vi.mocked(apiClient.get).mockImplementation((path: string) => { + if (path === "/roi-calculator/settings") { + return Promise.resolve(publicSettings); + } + if (path === "/roi-calculator/report") return Promise.resolve({ report: summary }); + return Promise.resolve(idleStatus); + }); + render(); + fireEvent.click(await screen.findByRole("button", { name: "Settings" })); + const load = screen.getByRole("button", { name: "Load repositories" }); + if (provider === "github") expect(load).toBeDisabled(); + else expect(load).toBeEnabled(); + expect(screen.getByRole("textbox", { name: "Repository name" })).toBeEnabled(); + }); + + it("closes the old settings dialog when saving a different source", async () => { + const gitlabSettings = { + ...settings, + source_provider: "gitlab", + gitlab_api_url: "https://gitlab.com/api/v4", + has_gitlab_token: false, + repos: [], + ready: false, + }; + vi.mocked(apiClient.put).mockResolvedValue(gitlabSettings); + render(); + fireEvent.click(await screen.findByRole("button", { name: "Settings" })); + fireEvent.change(screen.getByLabelText("Repository source"), { target: { value: "gitlab" } }); + fireEvent.click(screen.getByRole("button", { name: "Save settings" })); + expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeVisible(); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + expect(screen.getAllByLabelText("Repository source")).toHaveLength(1); + expect(screen.getByLabelText("Repository source")).toHaveValue("gitlab"); + }); + + it("keeps a running analysis visible when a source change finishes saving", async () => { + const gitlabSettings = { ...settings, source_provider: "gitlab", repos: ["group/project"] }; + const runningStatus = { ...idleStatus, running: true, phase: "estimating", total: 1 }; + const saveRequest = Promise.withResolvers(); + vi.mocked(apiClient.put).mockReturnValue(saveRequest.promise); + render(); + fireEvent.click(await screen.findByRole("button", { name: "Settings" })); + fireEvent.change(screen.getByLabelText("Repository source"), { target: { value: "gitlab" } }); + fireEvent.click(screen.getByRole("button", { name: "Save settings" })); + vi.mocked(apiClient.get).mockResolvedValue(runningStatus); + expect(await screen.findByRole("progressbar", { hidden: true }, { timeout: 3000 })).toBeInTheDocument(); + + saveRequest.resolve(gitlabSettings); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + expect(screen.getByRole("progressbar", { name: "Sync progress" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Cancel sync" })).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + }); + + it.each(["success", "failure"])("ignores an old report refresh %s after switching sources", async (outcome) => { + const gitlabSettings = { ...settings, source_provider: "gitlab", repos: [], ready: false }; + const oldRequest = Promise.withResolvers<{ report: typeof summary }>(); + const complete = { ...idleStatus, phase: "complete", finished_at: "2026-09-30T12:00:00Z" }; + vi.mocked(apiClient.put).mockResolvedValue(gitlabSettings); + render(); + fireEvent.click(await screen.findByRole("button", { name: "Settings" })); + vi.mocked(apiClient.get) + .mockClear() + .mockImplementation((path: string) => + path === "/roi-calculator/report" ? oldRequest.promise : Promise.resolve(complete), + ); + await waitFor( + () => expect(apiClient.get).toHaveBeenCalledWith("/roi-calculator/report", { accessToken: "token" }), + { + timeout: 3000, + }, + ); + fireEvent.change(screen.getByLabelText("Repository source"), { target: { value: "gitlab" } }); + fireEvent.click(screen.getByRole("button", { name: "Save settings" })); + expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeVisible(); + + await act(async () => { + if (outcome === "success") oldRequest.resolve({ report: summary }); + else oldRequest.reject(new Error("The old source is unavailable")); + }); + expect(screen.getByRole("heading", { name: "Connect your repositories" })).toBeVisible(); + expect(screen.getByLabelText("Repository source")).toHaveValue("gitlab"); + expect(screen.queryByText("Improve request routing")).not.toBeInTheDocument(); + expect(screen.queryByText("The old source is unavailable")).not.toBeInTheDocument(); + + const nextReport = { ...summary, pulls: [{ ...summary.pulls[0], title: "New source merge request" }] }; + vi.mocked(apiClient.get).mockImplementation((path: string) => + Promise.resolve( + path === "/roi-calculator/report" + ? { report: nextReport } + : { ...complete, finished_at: "2026-09-30T13:00:00Z" }, + ), + ); + expect(await screen.findByText("New source merge request", {}, { timeout: 3000 })).toBeVisible(); + expect(screen.queryByText("Improve request routing")).not.toBeInTheDocument(); }); it("clearly identifies the sample report and returns to setup when exiting", async () => { @@ -263,18 +381,212 @@ describe("ROICalculatorView", () => { fireEvent.click(await screen.findByRole("button", { name: "Preview sample report" })); expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); - expect(screen.getByText("Spend per estimated engineering hour")).toBeVisible(); + expect(screen.getByRole("tab", { name: "Branches" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByText("Cost / estimated hour")).toBeVisible(); expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); expect(screen.queryByRole("tab", { name: "Settings" })).not.toBeInTheDocument(); fireEvent.click(screen.getByRole("button", { name: "Exit demo" })); - expect(screen.getByRole("heading", { name: "Connect GitHub to get started" })).toBeVisible(); + expect(screen.getByRole("heading", { name: "Connect your repositories" })).toBeVisible(); expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); expect(apiClient.post).not.toHaveBeenCalled(); expect(apiClient.put).not.toHaveBeenCalled(); }); + it("opens sample PR costs from a live report and restores the live data on exit", async () => { + const samplePull = { + ...summary.pulls[0], + title: "Sample usage breakdown", + source_repo: "github.com/org/repo", + source_branch: "feature/usage", + branch_cost: { + status: "matched", + spend: 9.1, + requests: 75, + repo: "github.com/org/repo", + branch: "feature/usage", + }, + }; + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (path === "/roi-calculator/report") { + return Promise.resolve({ + report: options?.query?.mode === "demo" ? { ...summary, mode: "demo", pulls: [samplePull] } : summary, + }); + } + return Promise.resolve(idleStatus); + }); + + render(); + fireEvent.click(await screen.findByRole("tab", { name: "Branches" })); + fireEvent.change(screen.getByRole("searchbox"), { target: { value: "no matching PR" } }); + fireEvent.click(screen.getByRole("button", { name: "Preview sample report" })); + + expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); + expect(window.location.search).toBe("?demo=1"); + expect(screen.getByRole("tab", { name: "Branches" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByRole("searchbox")).toHaveValue(""); + expect(screen.getByRole("cell", { name: "$9.10" })).toBeVisible(); + expect(screen.queryByText("Improve request routing")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); + + const runningStatus = { ...idleStatus, running: true, phase: "estimating", total: 1 }; + vi.mocked(apiClient.get).mockClear().mockResolvedValue(runningStatus); + await waitFor(() => expect(apiClient.get).toHaveBeenCalledWith("/roi-calculator/sync", { accessToken: "token" }), { + timeout: 3000, + }); + expect(screen.queryByRole("progressbar")).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Open estimate for org/repo pull request 42" })); + expect(await screen.findByRole("dialog")).toBeVisible(); + expect(screen.getByText("75 requests")).toBeVisible(); + expect(screen.getByText(/branch:feature\/usage/)).toBeVisible(); + fireEvent.click(screen.getByRole("button", { name: "Close" })); + fireEvent.click(screen.getByRole("button", { name: "Exit demo" })); + + expect(screen.getByText("Improve request routing")).toBeVisible(); + expect(screen.queryByText("Sample usage breakdown")).not.toBeInTheDocument(); + expect(window.location.search).toBe(""); + expect(screen.getByRole("button", { name: "Syncing…" })).toBeDisabled(); + expect(apiClient.post).not.toHaveBeenCalled(); + expect(apiClient.put).not.toHaveBeenCalled(); + }); + + it("opens a demo link with sample data even while live analysis is running", async () => { + window.history.replaceState(null, "", "/roi-calculator/?demo=1"); + const demoSummary = { ...summary, mode: "demo", metrics: { ...summary.metrics, total_spend: 38.4 } }; + const runningStatus = { ...idleStatus, running: true, phase: "estimating", total: 1 }; + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (path === "/roi-calculator/report") { + return Promise.resolve({ report: options?.query?.mode === "demo" ? demoSummary : summary }); + } + return Promise.resolve(runningStatus); + }); + render(); + expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); + expect(screen.getByText("$38.40")).toBeVisible(); + expect(screen.queryByRole("progressbar")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Exit demo" })); + expect(screen.getByRole("progressbar")).toBeVisible(); + expect(screen.getByText("$20.00")).toBeVisible(); + expect(window.location.search).toBe(""); + }); + + it.each(["report", "sync"])("loads a demo link when the live %s request fails", async (failedRequest) => { + window.history.replaceState(null, "", "/roi-calculator/?demo=1"); + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (options?.query?.mode === "demo") return Promise.resolve({ report: { ...summary, mode: "demo" } }); + if (path === `/roi-calculator/${failedRequest}`) return Promise.reject(new Error("Live data unavailable")); + if (path === "/roi-calculator/report") return Promise.resolve({ report: summary }); + return Promise.resolve(idleStatus); + }); + render(); + expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); + expect(screen.getByText("Gateway AI cost")).toBeVisible(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + }); + + it("shows the demo without waiting for a stalled live request", async () => { + window.history.replaceState(null, "", "/roi-calculator/?demo=1"); + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (options?.query?.mode === "demo") return Promise.resolve({ report: { ...summary, mode: "demo" } }); + return new Promise(() => {}); + }); + render(); + expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); + expect(screen.getByText("Gateway AI cost")).toBeVisible(); + }); + + it("keeps the live calculator usable when a demo link cannot load sample data", async () => { + window.history.replaceState(null, "", "/roi-calculator/?demo=1&from=review#overview"); + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (options?.query?.mode === "demo") return Promise.reject(new Error("Sample data unavailable")); + if (path === "/roi-calculator/report") return Promise.resolve({ report: summary }); + return Promise.resolve(idleStatus); + }); + render(); + expect(await screen.findByText("Gateway AI cost")).toBeVisible(); + expect(screen.getByText("$20.00")).toBeVisible(); + expect(screen.getByRole("alert")).toHaveTextContent("Sample data unavailable"); + expect(screen.getByRole("button", { name: "Settings" })).toBeEnabled(); + expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + expect(window.location.search).toBe("?from=review"); + expect(window.location.hash).toBe("#overview"); + }); + + it.each(["report", "sync"])("waits for the live %s when exiting a demo", async (pendingRequest) => { + window.history.replaceState(null, "", "/roi-calculator/?demo=1"); + const pending = Promise.withResolvers<{ report: typeof summary } | typeof idleStatus>(); + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (options?.query?.mode === "demo") return Promise.resolve({ report: { ...summary, mode: "demo" } }); + if (path === `/roi-calculator/${pendingRequest}`) return pending.promise; + if (path === "/roi-calculator/report") return Promise.resolve({ report: summary }); + return Promise.resolve(idleStatus); + }); + render(); + fireEvent.click(await screen.findByRole("button", { name: "Exit demo" })); + expect(screen.getByText("Loading ROI Calculator…")).toBeVisible(); + expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); + expect(window.location.search).toBe(""); + + pending.resolve(pendingRequest === "report" ? { report: summary } : idleStatus); + expect(await screen.findByText("Gateway AI cost")).toBeVisible(); + expect(screen.getByText("$20.00")).toBeVisible(); + expect(screen.queryByText("Loading ROI Calculator…")).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + }); + + it.each(["report", "demo"])("retains a failed %s load after a successful sync poll", async (failedRequest) => { + if (failedRequest === "demo") window.history.replaceState(null, "", "/roi-calculator/?demo=1"); + const message = `The ${failedRequest} is unavailable`; + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (options?.query?.mode === "demo") return Promise.reject(new Error(message)); + if (path === "/roi-calculator/report") { + return failedRequest === "report" ? Promise.reject(new Error(message)) : Promise.resolve({ report: summary }); + } + return Promise.resolve(idleStatus); + }); + render(); + expect(await screen.findByRole("alert")).toHaveTextContent(message); + const running = { ...idleStatus, running: true, phase: "estimating", total: 1 }; + vi.mocked(apiClient.get).mockImplementation((path: string) => { + if (path === "/roi-calculator/report") return Promise.reject(new Error(message)); + return Promise.resolve(running); + }); + expect(await screen.findByRole("progressbar", { name: "Sync progress" }, { timeout: 3000 })).toBeVisible(); + expect(screen.getByRole("alert")).toHaveTextContent(message); + }); + + it("retries a failed initial report and clears its error only when the report recovers", async () => { + vi.mocked(apiClient.get).mockImplementation((path: string) => { + if (path === "/roi-calculator/settings") return Promise.resolve(settings); + if (path === "/roi-calculator/report") return Promise.reject(new Error("Report unavailable")); + return Promise.resolve(idleStatus); + }); + render(); + expect(await screen.findByRole("alert")).toHaveTextContent("Report unavailable"); + expect(screen.getByRole("button", { name: "Settings" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Run analysis" })).toBeEnabled(); + vi.mocked(apiClient.get).mockImplementation((path: string) => + Promise.resolve(path === "/roi-calculator/report" ? { report: summary } : idleStatus), + ); + expect(await screen.findByText("Gateway AI cost", {}, { timeout: 3000 })).toBeVisible(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument(); + }); + it("returns to Overview and shows the last sync time when completion is polled from Settings", async () => { const runningStatus = { ...idleStatus, @@ -299,10 +611,9 @@ describe("ROICalculatorView", () => { render(); expect(await screen.findByRole("progressbar", { name: "Sync progress" })).toBeInTheDocument(); - expect(await screen.findByText("Spend per estimated engineering hour", {}, { timeout: 5000 })).toBeInTheDocument(); - expect(screen.queryByRole("heading", { name: "Connect GitHub to get started" })).not.toBeInTheDocument(); + expect(await screen.findByText("Gateway AI cost", {}, { timeout: 5000 })).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument(); expect(screen.getByRole("status")).toHaveTextContent("Last synced Sep 30, 2026, 12:00 PM UTC"); - expect(screen.getByRole("status")).toHaveTextContent("57 of 57 estimates reused"); }); it("shows the sync error returned by the status endpoint", async () => { @@ -380,14 +691,14 @@ describe("ROICalculatorView", () => { expect(await screen.findByRole("alert", {}, { timeout: 5000 })).toHaveTextContent( "The sync status could not be loaded.", ); - expect(await screen.findByText("Spend per estimated engineering hour", {}, { timeout: 7000 })).toBeInTheDocument(); + expect(await screen.findByText("Gateway AI cost", {}, { timeout: 7000 })).toBeInTheDocument(); expect(screen.queryByText("The sync status could not be loaded.")).not.toBeInTheDocument(); }); it("saves the edited schedule before running from Settings", async () => { vi.mocked(apiClient.put).mockResolvedValue(settings); vi.mocked(apiClient.post).mockResolvedValue({ ...idleStatus, running: true }); render(); - fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(await screen.findByRole("button", { name: "Settings" })); fireEvent.change(screen.getByLabelText("Update interval (hours)"), { target: { value: "6" } }); fireEvent.click(screen.getByRole("button", { name: "Save and run analysis" })); await waitFor(() => expect(apiClient.post).toHaveBeenCalledWith("/roi-calculator/sync", { accessToken: "token" })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx index 2b17d2019ee..f9f467c7114 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx @@ -1,8 +1,8 @@ "use client"; -import { Page } from "@/components/shared/Page"; +import { Page, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; import React from "react"; -import { Calculator, RefreshCw } from "lucide-react"; +import { Calculator, RefreshCw, Settings2 } from "lucide-react"; import { apiClient } from "@/components/networking"; import { DemoNotice } from "@/components/shared/DemoNotice"; @@ -11,12 +11,13 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Tabs } from "@/components/ui/tabs"; +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog"; import { extractErrorMessage } from "@/utils/errorUtils"; import { isProxyAdminTierRole } from "@/utils/roles"; import ROISettingsPanel from "./ROISettingsPanel"; import { IdentityMatchDialog, type PersonMatchSelection, PullReasoningDialog } from "./ROICalculatorDialogs"; -import { ROIOverview, ROIPeopleView } from "./ROICalculatorViews"; +import { ROIBranches, ROIOverview, ROIPeopleView } from "./ROICalculatorViews"; import { filterPulls, formatSyncedAt } from "./roiCalculatorData"; import type { ROIIdentityMapResponse, @@ -28,7 +29,7 @@ import type { ROISyncStatus, } from "./roiCalculatorData"; -type View = "overview" | "people" | "settings"; +type View = "overview" | "people" | "branches"; const IDLE_STATUS: ROISyncStatus = { running: false, @@ -43,6 +44,13 @@ const IDLE_STATUS: ROISyncStatus = { error: null, }; +function updateDemoUrl(enabled: boolean) { + const url = new URL(window.location.href); + if (enabled) url.searchParams.set("demo", "1"); + else url.searchParams.delete("demo"); + window.history.replaceState(null, "", url); +} + export default function ROICalculatorView({ accessToken, userRole = null, @@ -55,6 +63,7 @@ export default function ROICalculatorView({ const [sampleSummary, setSampleSummary] = React.useState(null); const adminReadOnly = isViewOnly && isProxyAdminTierRole(userRole ?? ""); const readOnly = adminReadOnly || sampleSummary !== null; + const [settingsOpen, setSettingsOpen] = React.useState(false); const [view, setView] = React.useState("overview"); const [settings, setSettings] = React.useState(null); const [liveSummary, setSummary] = React.useState(null); @@ -63,8 +72,16 @@ export default function ROICalculatorView({ const [selectedPull, setSelectedPull] = React.useState(null); const [matchingPerson, setMatchingPerson] = React.useState(null); const [error, setError] = React.useState(null); + const [demoError, setDemoError] = React.useState(null); + const [reportError, setReportError] = React.useState(null); + const [syncError, setSyncError] = React.useState(null); + const [loadingInitialData, setLoadingInitialData] = React.useState(true); + const [loadingLiveData, setLoadingLiveData] = React.useState(true); const statusRef = React.useRef(IDLE_STATUS); - const settingsLoaded = settings !== null; + const reportNeedsRefresh = React.useRef(false); + const sourceRevision = React.useRef(0); + const settingsLoaded = settings !== null && !loadingInitialData && !loadingLiveData; + const requestError = [error, demoError, reportError, syncError].filter(Boolean).join(" "); const [query, setQuery] = React.useState(""); const loadReport = React.useCallback(async () => { @@ -76,21 +93,63 @@ export default function ROICalculatorView({ React.useEffect(() => { if (!accessToken) return; let cancelled = false; - Promise.all([ - apiClient.get("/roi-calculator/settings", { accessToken }), - apiClient.get("/roi-calculator/report", { accessToken }), - apiClient.get("/roi-calculator/sync", { accessToken }), - ]) - .then(([nextSettings, reportResponse, syncStatus]) => { + const demoRequested = new URLSearchParams(window.location.search).get("demo") === "1"; + const settingsRequest = apiClient.get("/roi-calculator/settings", { accessToken }); + const reportRequest = apiClient + .get("/roi-calculator/report", { accessToken }) + .then((response) => { + if (cancelled) return; + setSummary(response.report); + setReportError(null); + reportNeedsRefresh.current = false; + }) + .catch((reason: unknown) => { + if (cancelled) return; + setReportError(extractErrorMessage(reason)); + reportNeedsRefresh.current = true; + }); + const statusRequest = apiClient + .get("/roi-calculator/sync", { accessToken }) + .then((syncStatus) => { if (cancelled) return; - setSettings(nextSettings); - setSummary(reportResponse.report); setStatus(syncStatus); statusRef.current = syncStatus; + setSyncError(null); + }) + .catch((reason: unknown) => { + if (!cancelled) setSyncError(extractErrorMessage(reason)); + }); + const liveData = Promise.all([reportRequest, statusRequest]) + .then(() => null) + .finally(() => { + if (!cancelled) setLoadingLiveData(false); + }); + Promise.all([ + settingsRequest, + demoRequested + ? apiClient + .get("/roi-calculator/report", { accessToken, query: { mode: "demo" } }) + .catch((reason: unknown) => { + if (!cancelled) { + setDemoError(`Could not load demo data: ${extractErrorMessage(reason)}`); + updateDemoUrl(false); + } + return liveData; + }) + : liveData, + ]) + .then(([nextSettings, sampleResponse]) => { + if (cancelled) return; + setSettings(nextSettings); + setSampleSummary(sampleResponse?.report ?? null); setError(null); + if (sampleResponse) setDemoError(null); }) .catch((reason: unknown) => { if (!cancelled) setError(extractErrorMessage(reason)); + }) + .finally(() => { + if (!cancelled) setLoadingInitialData(false); }); return () => { cancelled = true; @@ -101,31 +160,36 @@ export default function ROICalculatorView({ if (!accessToken || !settingsLoaded) return; let cancelled = false; let requestInFlight = false; - let reportNeedsRefresh = false; const interval = window.setInterval(() => { if (requestInFlight) return; requestInFlight = true; + const revision = sourceRevision.current; + const isCurrent = () => !cancelled && revision === sourceRevision.current; apiClient .get("/roi-calculator/sync", { accessToken }) .then(async (nextStatus) => { - if (cancelled) return; + if (!isCurrent()) return; const previousStatus = statusRef.current; statusRef.current = nextStatus; setStatus(nextStatus); + setSyncError(null); const finished = !nextStatus.running && nextStatus.phase === "complete"; const reportChanged = previousStatus.running || nextStatus.finished_at !== previousStatus.finished_at; - if (finished && (reportChanged || reportNeedsRefresh)) { - reportNeedsRefresh = true; - const report = await loadReport(); - if (cancelled) return; - setSummary(report); - reportNeedsRefresh = false; - setView((current) => (current === "settings" ? "overview" : current)); + if (reportNeedsRefresh.current || (finished && reportChanged)) { + reportNeedsRefresh.current = true; + try { + const report = await loadReport(); + if (!isCurrent()) return; + setSummary(report); + setReportError(null); + reportNeedsRefresh.current = false; + } catch (reason: unknown) { + if (isCurrent()) setReportError(extractErrorMessage(reason)); + } } - if (!cancelled) setError(null); }) .catch((reason: unknown) => { - if (!cancelled) setError(extractErrorMessage(reason)); + if (isCurrent()) setSyncError(extractErrorMessage(reason)); }) .finally(() => { requestInFlight = false; @@ -144,6 +208,8 @@ export default function ROICalculatorView({ const nextStatus = await apiClient.post("/roi-calculator/sync", { accessToken }); statusRef.current = nextStatus; setStatus(nextStatus); + setSyncError(null); + setSettingsOpen(false); } catch (reason) { setError(extractErrorMessage(reason)); } @@ -152,7 +218,11 @@ export default function ROICalculatorView({ const cancelSync = React.useCallback(async () => { if (!accessToken || readOnly) return; try { - setStatus(await apiClient.delete("/roi-calculator/sync", { accessToken })); + const nextStatus = await apiClient.delete("/roi-calculator/sync", { accessToken }); + setStatus(nextStatus); + statusRef.current = nextStatus; + setSyncError(null); + setError(null); } catch (reason) { setError(extractErrorMessage(reason)); } @@ -166,6 +236,8 @@ export default function ROICalculatorView({ body: payload, }); setSummary(response.report); + setReportError(null); + reportNeedsRefresh.current = false; setSettings((current) => (current ? { ...current, identity_map: response.identity_map } : current)); }, [accessToken, readOnly], @@ -173,7 +245,7 @@ export default function ROICalculatorView({ const filteredPulls = React.useMemo(() => (summary ? filterPulls(summary.pulls, query) : []), [query, summary]); - if (error && !settings) { + if (error && !settings && !loadingInitialData) { return (
@@ -184,9 +256,11 @@ export default function ROICalculatorView({ ); } - if (!settings) { + const awaitingLiveData = !sampleSummary && loadingLiveData; + if (!settings || loadingInitialData || awaitingLiveData) { return (
+

Loading ROI Calculator…

@@ -200,58 +274,86 @@ export default function ROICalculatorView({ query: { mode: "demo" }, }); setSampleSummary(response.report); - setView("overview"); + setDemoError(null); + updateDemoUrl(true); + setView("branches"); + setQuery(""); } catch (reason) { - setError(extractErrorMessage(reason)); + setDemoError(`Could not load demo data: ${extractErrorMessage(reason)}`); } }; - const resetView = (updated: ROISettings) => { + const resetView = (updated: ROISettings, resetSyncStatus = true) => { + sourceRevision.current += 1; setSettings(updated); setSummary(null); + setReportError(null); + reportNeedsRefresh.current = false; + setSettingsOpen(false); setView("overview"); - setStatus(IDLE_STATUS); - statusRef.current = IDLE_STATUS; + if (resetSyncStatus) { + setStatus(IDLE_STATUS); + statusRef.current = IDLE_STATUS; + setSyncError(null); + setError(null); + } }; const showLiveStatus = !sampleSummary && !status.running; - const scheduleLabel = settings.update_interval_minutes ? "Automatic updates enabled" : "Manual updates"; + const showReportActions = summary !== null || reportError !== null; const progress = status.total > 0 ? Math.min(100, (status.done / status.total) * 100) : 0; const statusIsIdleOrComplete = status.phase === "idle" || status.phase === "complete"; const syncIsUpToDate = !status.running && statusIsIdleOrComplete; - const syncedAt = syncIsUpToDate ? summary?.synced_at : null; + const syncedAt = sampleSummary?.synced_at ?? (syncIsUpToDate ? summary?.synced_at : null); return ( - - + +
ROI Calculator - {!liveSummary && showLiveStatus && ( - + {!sampleSummary && ( +
+ {showLiveStatus && ( + + )} + {showReportActions && ( + + )} + {showReportActions && !readOnly && ( + + )} +
)}
- - {summary - ? `${summary.start} through ${summary.end} · UTC` - : "Compare gateway spend with estimated engineering effort for merged pull requests"} + + + {summary + ? `${summary.start} through ${summary.end} · UTC` + : "Compare AI costs with estimated engineering effort"} + {syncedAt && ( - + Last synced {formatSyncedAt(syncedAt)} - {!status.running && status.phase === "complete" && status.reused > 0 - ? ` · ${status.reused} of ${status.total} estimates reused` - : ""} )} - {sampleSummary && setSampleSummary(null)} />} - {liveSummary && showLiveStatus && ( -

- {status.next_update ? `Next update ${formatSyncedAt(status.next_update)}` : scheduleLabel} -

+ {sampleSummary && ( + { + updateDemoUrl(false); + setSampleSummary(null); + }} + /> )} {adminReadOnly && (

@@ -260,30 +362,22 @@ export default function ROICalculatorView({ )} {summary && ( -

- setView(value as View)}> - - Overview - People - {!sampleSummary && Settings} - - - {view !== "settings" && !readOnly && ( - - )} -
+ setView(value as View)}> + + Overview + People + Branches + + )} - {error && ( + {!sampleSummary && requestError && ( ROI Calculator request failed - {error} + {requestError} )} - {status.error && ( + {!sampleSummary && status.error && ( Sync failed {status.error} @@ -295,9 +389,9 @@ export default function ROICalculatorView({ {warning} ))} - {status.running && ( + {!sampleSummary && status.running && ( - +

- {status.done} of {status.total} pull requests processed · {status.reused} reused + {status.done} of {status.total} changes processed · {status.reused} reused {` · ${status.elapsed_seconds ?? 0}s elapsed`} {status.remaining_seconds != null ? ` · about ${status.remaining_seconds}s remaining` : ""}

@@ -325,7 +419,7 @@ export default function ROICalculatorView({ )} - {view === "settings" || (!summary && !status.running) ? ( + {!summary && !status.running && !reportError ? ( setView("people")} + onViewBranches={() => setView("branches")} + /> + )} + {view === "branches" && summary && ( + setView("people")} /> )} {view === "people" && summary && ( @@ -355,6 +456,34 @@ export default function ROICalculatorView({ readOnly={readOnly} /> )} + + + + Calculator settings + Connect repositories and choose how to estimate effort. + + { + if ( + updated.source_provider !== settings.source_provider || + updated.github_api_url !== settings.github_api_url || + updated.gitlab_api_url !== settings.gitlab_api_url + ) { + resetView(updated, false); + return; + } + setSettings(updated); + }} + onReset={resetView} + onStartSync={startSync} + readOnly={readOnly} + syncDisabled={status.running} + /> + + setSelectedPull(null)} /> {!readOnly && ( void }; export function ROIOverview({ summary, - pulls, - query, - onQueryChange, onSelectPull, onViewPeople, -}: { - summary: ROISummary; - pulls: ROIPull[]; - query: string; - onQueryChange: (value: string) => void; - onSelectPull: (pull: ROIPull) => void; + onViewBranches, +}: PullSelection & { onViewPeople: () => void; + onViewBranches: () => void; }) { - const [pagination, setPagination] = React.useState({ query, visibleCount: 10 }); - const visibleCount = pagination.query === query ? pagination.visibleCount : 10; const metrics = summary.metrics; - const unavailableRate = - metrics.output_hours > 0 - ? "Spend per estimated hour is unavailable until all selected repositories can be read." - : "A rate requires matched estimated hours greater than zero and access to all selected repositories."; + const branches = summary.branch_metrics; + const topPulls = highestCostPulls(summary.pulls); return ( -
-
- - - - -
-

- {formatMoney(metrics.excluded_spend)} of {formatMoney(metrics.total_spend)} total gateway spend is excluded from - the matched cohort. -

-
- Calculation details -
-

- {metrics.cost_per_hour != null - ? `${formatMoney(metrics.matched_spend)} gateway spend ÷ ${formatNumber(metrics.output_hours)} estimated engineering hours = ${formatMoney(metrics.cost_per_hour)} per estimated hour.` - : unavailableRate} -

-

- The comparison includes {metrics.cohort_people} matched {metrics.cohort_people === 1 ? "person" : "people"}{" "} - with complete PR estimates, for the same period in UTC. {metrics.matched_prs} of {metrics.merged_prs} PRs - have email matches. {formatMoney(metrics.excluded_spend)} of {formatMoney(metrics.total_spend)} total - gateway spend is excluded. -

-

- Gateway spend includes all of each person’s usage, across repositories. This does not measure hours saved by - AI or financial returns. -

- -
-
- - - - Spend and estimated engineering effort - - Daily matched gateway spend and estimated engineering hours for the same UTC period - - - - - - - - formatMoney(Number(value))} /> - - } /> - } /> - - - - - - - - - -
- Pull requests - - {metrics.merged_prs} merged · {metrics.estimated_prs} estimated · {metrics.pending_prs} need attention - -
- onQueryChange(event.target.value)} +
+
+
+ - - - - - - Pull request - Estimated hours + + + + + +
+
+

Where AI costs are matched

+

+ People use gateway account costs. Branches use tagged requests for these repositories. +

+
+
+
+ + + View + Matched cost + Unmatched cost + Changes matched - {pulls.slice(0, visibleCount).map((pull) => ( - - - - - {estimateLabel(pull.estimate)} - - ))} - {pulls.length === 0 && ( - - - {query ? "No matching pull requests." : "No merged pull requests in this period."} - - - )} + + + + + + {formatMoney(metrics.matched_spend)} + + + {formatMoney(metrics.excluded_spend)} + + + {metrics.matched_prs} / {metrics.merged_prs} + + + + + + + + {formatMoney(branches?.spend)} + + + {formatMoney(branches?.unlinked_spend)} + + + {branches?.matched_pulls ?? 0} / {metrics.merged_prs} + +
- {pulls.length > visibleCount && ( +
+
+ +
+ ); +} + +export function ROIBranches({ + summary, + onSelectPull, + pulls, + query, + onQueryChange, +}: PullSelection & { + pulls: ROIPull[]; + query: string; + onQueryChange: (query: string) => void; +}) { + return ( +
+
+
+ + +
+
+ +
+ ); +} + +function ROIPulls({ + summary, + onSelectPull, + pulls, + query = "", + onQueryChange, + compact = false, + onViewBranches, +}: PullSelection & { + pulls: ROIPull[]; + query?: string; + onQueryChange?: (query: string) => void; + compact?: boolean; + onViewBranches?: () => void; +}) { + const changeName = summary.source_provider === "gitlab" ? "merge request" : "pull request"; + const [pagination, setPagination] = React.useState({ query, visibleCount: 10 }); + const visibleCount = pagination.query === query ? pagination.visibleCount : 10; + const metrics = summary.metrics; + const emptyMessage = compact + ? "No merged changes with tagged costs yet. Open Branches to see how to add tags." + : `No merged ${changeName}s in this period.`; + return ( +
+
+
+

{compact ? "Highest-cost changes" : "Costs by branch"}

+

+ {compact + ? "Merged work ranked by tagged AI cost" + : `${metrics.merged_prs} ${changeName}s · ${metrics.estimated_prs} estimated`} + {!compact && metrics.pending_prs > 0 && ( + · {metrics.pending_prs} need attention + )} +

+
+ {compact ? ( + + ) : ( +
+
+ )} +
+
+ + + + + {changeName === "merge request" ? "Merge request" : "Pull request"} + + AI cost + Estimated effort + + + + {pulls.slice(0, visibleCount).map((pull) => ( + + + + + + {branchCostLabel(pull)} + + {estimateLabel(pull.estimate)} + + ))} + {pulls.length === 0 && ( + + + {query ? `No matching ${changeName}s. Try another search.` : emptyMessage} + + + )} + +
+ {pulls.length > visibleCount && ( +
+

+ Showing {Math.min(visibleCount, pulls.length)} of {pulls.length} +

- )} - - - +
+ )} +
+
+ ); +} + +function ROIMetrics({ summary, branchMode }: { summary: ROISummary; branchMode: boolean }) { + const branches = summary.branch_metrics; + const metrics = summary.metrics; + return ( +
+ + + + +
+ ); +} + +function MetricCard({ + title, + value, + description, + primary = false, +}: { + title: string; + value: string; + description: string; + primary?: boolean; +}) { + return ( +
+
{title}
+
+ {value} +
+
{description}
); } -function MetricCard({ title, value }: { title: string; value: string }) { +function ROIComparison({ + summary, + branchMode, + onViewPeople, +}: { + summary: ROISummary; + branchMode: boolean; + onViewPeople?: () => void; +}) { + const branches = summary.branch_metrics; + const metrics = summary.metrics; + const unavailableRate = + metrics.output_hours > 0 + ? "Spend per estimated hour is unavailable until all selected repositories can be read." + : "Match gateway accounts to calculate costs per estimated hour."; return ( - - - {title} - {value} - - +
+ {!branchMode && metrics.cohort_people === 0 && onViewPeople && ( +
+

Match people to gateway accounts to see their AI costs.

+ +
+ )} +
+ + + + + {formatMoney(branchMode ? branches?.unlinked_spend : metrics.excluded_spend)}{" "} + {branchMode ? "in unmatched costs" : "excluded from calculation"} + + +
+

{effortNote(summary.effort_basis)}

+ {branchMode ? ( + <> +

+ Only branches with matched request costs and complete effort estimates enter the calculation. Costs + cover retained requests in this report’s UTC dates, not the branch’s lifetime. +

+

+ Open a change below to find its repository and branch tags. Send both with each gateway request. Email + matching is not required. Shared branches stay ambiguous so their costs are not counted twice. +

+

+ {formatMoney(branches?.total_tagged_spend)} in tagged costs was found for these repositories. Costs + without a unique, fully estimated change stay unmatched. +

+ {(summary.unlinked_branches?.length ?? 0) > 0 && ( +
+

Unmatched branches

+
    + {summary.unlinked_branches?.map((row) => ( +
  • + + {row.repo} +
    + {row.branch} +     + {formatMoney(row.spend)} +
    • + ))} +
+
+ )} + + ) : ( + <> +

+ {metrics.cost_per_hour != null + ? `${formatMoney(metrics.matched_spend)} AI costs ÷ ${formatNumber(metrics.output_hours)} estimated hours = ${formatMoney(metrics.cost_per_hour)} per estimated hour.` + : unavailableRate} +

+

+ Includes {metrics.cohort_people} matched {metrics.cohort_people === 1 ? "person" : "people"} with + complete estimates. Costs include each person’s full gateway usage across repositories during this UTC + period. +

+ {onViewPeople && ( + + )} + + )} +
+
+
); } @@ -221,93 +456,110 @@ export function ROIPeopleView({ }; return (
-
+
+ + +
+
+
+

People and account matches

+

Select a person to match their gateway email.

+
-

- {effortNote(summary.effort_basis)} Spend includes each person’s full gateway usage for this period. This does - not measure hours saved by AI or financial returns. -

- - - - - - Person - Gateway spend - Estimated hours - Spend / estimated hour - - - - {summary.people.map((person) => ( - - -
- {person.logins.length ? ( - person.logins.map((login) => - readOnly ? ( - {login} - ) : ( - - ), - ) - ) : ( - Unassigned gateway spend - )} +
+
+ + + Person + AI cost + Estimated effort + Cost / est. hour + + + + {summary.people.map((person) => ( + + +
+ {person.logins.length ? ( + person.logins.map((login) => + readOnly ? ( + {login} + ) : ( + + ), + ) + ) : ( + Unassigned gateway spend + )} + {person.match_methods.some( (method) => ["manual", "commit email", "profile email"].includes(method) && person.spend != null, - ) ? ( - Matched - ) : ( - Unmatched - )} -
-

{person.email || "Email unavailable"}

- {person.logins.some((login) => identityMap[login.toLowerCase()]) && ( -

Manual email match

- )} - {!person.eligible &&

Excluded from ratio

} - - {formatMoney(person.spend)} - - {person.estimated_prs > 0 ? `${formatNumber(person.hours)} hrs` : "—"} -

- {person.prs} {person.prs === 1 ? "PR" : "PRs"} - {person.pending_prs > 0 ? ` · ${person.pending_prs} pending` : ""} -

- - {formatMoney(person.cost_per_hour)} - - ))} - {summary.people.length === 0 && ( - - - No people in this period. - - - )} - -
- - -
- How email matching works -

- Matches use the author’s public GitHub email or commit emails associated with their GitHub account. Email - matching ignores case. Private, noreply, and ambiguous emails stay unmatched. Manual matches take priority. - People with no spend record or incomplete PR estimates are excluded from the ratio. -

+ ) + ? "Matched" + : "Unmatched"} + +
+

+ {person.email || "No public email"} + {person.logins.some((login) => identityMap[login.toLowerCase()]) ? " · Manual match" : ""} +

+ + {formatMoney(person.spend)} + + {person.estimated_prs > 0 ? `${formatNumber(person.hours)} hrs` : "—"} +

+ {person.prs} {person.prs === 1 ? "change" : "changes"} + {person.pending_prs > 0 ? ` · ${person.pending_prs} pending` : ""} +

+ + + {formatMoney(person.cost_per_hour)} + {!person.eligible &&

Not included

} + + + ))} + {summary.people.length === 0 && ( + + + No people in this period. + + + )} + + +
+
+ + +
+

+ Matches use the author’s public profile email + {summary.source_provider === "gitlab" + ? "." + : " or commit emails associated with their GitHub account."}{" "} + Private, noreply, and ambiguous emails stay unmatched. Manual matches take priority. +

+

+ Costs include each person’s full gateway usage for this period. People without a spend record or with + incomplete estimates are not included in the calculation. +

+

{effortNote(summary.effort_basis)}

+
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROISettingsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROISettingsPanel.tsx index 977d0dbc760..540ac229de8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROISettingsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROISettingsPanel.tsx @@ -38,9 +38,13 @@ export default function ROISettingsPanel({ readOnly: boolean; syncDisabled: boolean; }) { - const initialStep = initialSettings.has_github_token ? 1 : 0; + const [provider, setProvider] = React.useState<"github" | "gitlab">(initialSettings.source_provider ?? "github"); + const sourceName = provider === "gitlab" ? "GitLab" : "GitHub"; + const savedToken = provider === "gitlab" ? initialSettings.has_gitlab_token : initialSettings.has_github_token; + const savedUrl = provider === "gitlab" ? initialSettings.gitlab_api_url : initialSettings.github_api_url; + const initialStep = savedToken ? 1 : 0; const [step, setStep] = React.useState(initialSettings.ready ? 2 : initialStep); - const [apiUrl, setApiUrl] = React.useState(initialSettings.github_api_url); + const [apiUrl, setApiUrl] = React.useState(savedUrl ?? "https://gitlab.com/api/v4"); const [token, setToken] = React.useState(""); const [clearToken, setClearToken] = React.useState(false); const [repos, setRepos] = React.useState(initialSettings.repos); @@ -62,8 +66,27 @@ export default function ROISettingsPanel({ const [error, setError] = React.useState(null); const [message, setMessage] = React.useState(null); - const canLoadRepositories = - initialSettings.has_github_token && !token.trim() && apiUrl === initialSettings.github_api_url; + const sourceUnchanged = provider === (initialSettings.source_provider ?? "github") && apiUrl === savedUrl; + const credentialsSaved = sourceUnchanged && !token.trim() && !clearToken; + const canLoadRepositories = credentialsSaved && (provider === "gitlab" || savedToken); + const tokenHelp = + provider === "gitlab" + ? "For private projects, use a token with read_api scope and project access." + : "For private repositories, use a token with read access to contents and pull requests."; + + const changeProvider = (next: "github" | "gitlab") => { + setProvider(next); + setApiUrl( + next === "gitlab" + ? initialSettings.gitlab_api_url ?? "https://gitlab.com/api/v4" + : initialSettings.github_api_url, + ); + setToken(""); + setClearToken(false); + setRepos([]); + setAvailableRepos([]); + setHasMoreRepos(false); + }; const loadRepositories = async (page: number) => { if (!accessToken || !canLoadRepositories) return; @@ -86,8 +109,10 @@ export default function ROISettingsPanel({ const saveSettings = async () => { if (!accessToken || readOnly) return false; + const tokenValue = clearToken ? null : token.trim() || undefined; const body: ROISettingsUpdate = { - github_api_url: apiUrl, + source_provider: provider, + ...(provider === "gitlab" ? { gitlab_api_url: apiUrl } : { github_api_url: apiUrl }), repos, estimator_model: model, estimator_prompt: prompt, @@ -95,8 +120,7 @@ export default function ROISettingsPanel({ update_interval_minutes: Number(intervalHours) * 60, ...(clearEstimatorKey ? { estimator_key: null } : {}), ...(estimatorKey.trim() ? { estimator_key: estimatorKey.trim() } : {}), - ...(clearToken ? { github_token: null } : {}), - ...(token.trim() ? { github_token: token.trim() } : {}), + ...(provider === "gitlab" ? { gitlab_token: tokenValue } : { github_token: tokenValue }), }; try { setBusy(true); @@ -122,6 +146,8 @@ export default function ROISettingsPanel({ event.preventDefault(); if (!(await saveSettings())) return; if (onboarding && step === 0) { + setStep(1); + if (!(token.trim() || savedToken) || clearToken) return; try { const result = await apiClient.get("/roi-calculator/repositories", { accessToken }); setAvailableRepos(result.repositories); @@ -157,7 +183,7 @@ export default function ROISettingsPanel({ try { const updated = await apiClient.post("/roi-calculator/setup/reset", { accessToken }); setRepos([]); - setStep(updated.has_github_token ? 1 : 0); + setStep(0); setResetOpen(false); onReset(updated); } catch (reason) { @@ -173,27 +199,25 @@ export default function ROISettingsPanel({ const formDisabled = busy || syncDisabled; const runDisabled = formDisabled || !repos.length || !model; - const githubUrlChanged = apiUrl !== initialSettings.github_api_url; - const missingReplacementToken = initialSettings.has_github_token && githubUrlChanged && !token.trim(); - const stepReady = [Boolean(token.trim() || initialSettings.has_github_token), repos.length > 0, Boolean(model)][step]; + const sourceUrlChanged = apiUrl !== savedUrl; + const missingReplacementToken = savedToken && sourceUrlChanged && !token.trim(); + const stepReady = [true, repos.length > 0, Boolean(model)][step]; const onboardingLabel = step < 2 ? "Continue" : "Start backfill"; const submitLabel = onboarding ? onboardingLabel : "Save settings"; return ( - - -

- {onboarding - ? ["Connect GitHub to get started", "Choose repositories", "Choose an estimator"][step] - : "ROI Calculator settings"} -

- - {onboarding - ? "Your gateway is already connected. Set up GitHub and an estimator to see your first report." - : "Choose GitHub repositories and the router model used for metadata-only estimates."} - - - + + {onboarding && ( + +

+ {["Connect your repositories", "Choose repositories", "Choose an estimator"][step]} +

+ + Your gateway is already connected. Choose a source and an estimator for your first report. + + + )} + {error && (

{error} @@ -205,16 +229,34 @@ export default function ROISettingsPanel({

)} {onboarding && ( -

Step {step + 1} of 3 · GitHub / Repositories / Estimator

+

Step {step + 1} of 3 · Source / Repositories / Estimator

)}
void submit(event)}>
{(!onboarding || step === 0) && ( - <> +
+ {!onboarding &&

Connection

} +
+ + + {provider !== (initialSettings.source_provider ?? "github") && !onboarding && ( +

+ Switching source starts a new report and resets email matches. +

+ )} +
- GitHub Enterprise settings + Self-hosted {sourceName}
- +
- +

- {initialSettings.has_github_token - ? "A token is saved securely and is never shown here." - : "Save a token to list repositories and read private repository metadata."} + {savedToken ? "A token is saved securely and is never shown here." : tokenHelp}

{missingReplacementToken && (

- Changing the GitHub API URL clears the saved token. Enter a replacement token to keep access. + Changing the API URL clears the saved token. Enter a replacement token to keep access.

)} - {initialSettings.has_github_token && ( + {savedToken && (
- +
)} {(!onboarding || step === 1) && ( -
- +
+

Repositories

+
{!canLoadRepositories && (

- Save the GitHub token and API URL before loading repositories. + {provider === "github" && !savedToken + ? "Save a GitHub token to browse repositories, or add a public repository by name." + : "Save the source and connection settings before loading repositories."}

)} {repos.length > 0 && ( @@ -302,12 +347,13 @@ export default function ROISettingsPanel({ ))}
)} -
- Add a repository by name +
+
setRepositoryName(e.target.value)} /> @@ -323,30 +369,27 @@ export default function ROISettingsPanel({ Add
-
-
- {availableRepos.map((repository) => ( - - ))} - {availableRepos.length === 0 && ( -

- Load repositories to choose which pull requests to analyze. -

- )}
+ {availableRepos.length > 0 && ( +
+ {availableRepos.map((repository) => ( + + ))} +
+ )} {hasMoreRepos && (
+ )} {(!onboarding || step === 2) && ( - <> +
+ {!onboarding &&

Estimation and updates

}
setBackfillDays(event.target.value)} - /> -
-
- - setIntervalHours(e.target.value)} - /> -

- 0 for manual updates; otherwise at least 5 minutes. Updates run while the gateway is running. -

+
+
+ + setBackfillDays(event.target.value)} + /> +
+
+ + setIntervalHours(e.target.value)} + /> +

+ Use 0 for manual updates. Automatic updates require at least 5 minutes and a running gateway. +

+
Advanced settings @@ -463,13 +509,13 @@ export default function ROISettingsPanel({

- Estimates use pull request metadata, without source code. Hours represent estimated effort without AI, - not measured hours saved. + Estimates use descriptions, file counts, and commit messages, without source code. Hours represent + estimated effort without AI, not measured hours saved.

- +
)} {!readOnly && ( -
+
{onboarding && step > 0 && (