feat(roi): add GitLab sources and branch cost attribution (#44324)
* feat(roi): support GitLab and tagged branch costs * fix(roi): count tagged branches independently of estimation status * test(roi): capture live GitHub and GitLab report validation * fix(roi): open estimate details at the start * fix(roi): clarify cost views and unify report layout * feat(roi): showcase per-PR costs in the sample report * fix(roi): separate report tabs and preserve branch cost attribution * fix(roi): preserve demo previews and align progress spacing * fix(roi): isolate demo loading and parallelize fork lookups Preserve active sync status when source changes finish saving, keep live reports available when demo requests fail, and cover each review regression * fix(roi): separate demo and live loading states Clear the demo URL on fallback, wait for live requests on exit, and retain request errors until the corresponding operation recovers * fix(roi): ignore refreshes from a previous source * fix: trust gateway context for ROI estimator exclusion * fix: preserve historical ROI estimator exclusion
BIN
.github/assets/roi-calculator-integrations/after-github.jpg
vendored
Normal file
|
After Width: | Height: | Size: 77 KiB |
BIN
.github/assets/roi-calculator-integrations/after-gitlab-detail-top.jpg
vendored
Normal file
|
After Width: | Height: | Size: 79 KiB |
BIN
.github/assets/roi-calculator-integrations/after-gitlab-detail.jpg
vendored
Normal file
|
After Width: | Height: | Size: 61 KiB |
BIN
.github/assets/roi-calculator-integrations/after-gitlab.jpg
vendored
Normal file
|
After Width: | Height: | Size: 86 KiB |
BIN
.github/assets/roi-calculator-integrations/before-github.jpg
vendored
Normal file
|
After Width: | Height: | Size: 75 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-exit-loading.jpg
vendored
Normal file
|
After Width: | Height: | Size: 40 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-fallback-live.jpg
vendored
Normal file
|
After Width: | Height: | Size: 88 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-overview.jpg
vendored
Normal file
|
After Width: | Height: | Size: 95 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-people.jpg
vendored
Normal file
|
After Width: | Height: | Size: 96 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-pr-costs.jpg
vendored
Normal file
|
After Width: | Height: | Size: 94 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-pr-detail.jpg
vendored
Normal file
|
After Width: | Height: | Size: 70 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-preview-link.jpg
vendored
Normal file
|
After Width: | Height: | Size: 82 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-with-live-errors.jpg
vendored
Normal file
|
After Width: | Height: | Size: 82 KiB |
BIN
.github/assets/roi-calculator-integrations/source-race-after.jpg
vendored
Normal file
|
After Width: | Height: | Size: 64 KiB |
BIN
.github/assets/roi-calculator-integrations/source-race-before.jpg
vendored
Normal file
|
After Width: | Height: | Size: 90 KiB |
|
|
@ -48960,6 +48960,121 @@
|
|||
"title": "HTTPValidationError",
|
||||
"type": "object"
|
||||
},
|
||||
"ROIBranchAttribution": {
|
||||
"properties": {
|
||||
"branch": {
|
||||
"title": "Branch",
|
||||
"type": "string"
|
||||
},
|
||||
"repo": {
|
||||
"title": "Repo",
|
||||
"type": "string"
|
||||
},
|
||||
"requests": {
|
||||
"default": 0,
|
||||
"title": "Requests",
|
||||
"type": "integer"
|
||||
},
|
||||
"spend": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Spend"
|
||||
},
|
||||
"status": {
|
||||
"default": "unattributed",
|
||||
"enum": [
|
||||
"matched",
|
||||
"unattributed",
|
||||
"ambiguous",
|
||||
"unavailable"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"repo",
|
||||
"branch"
|
||||
],
|
||||
"title": "ROIBranchAttribution",
|
||||
"type": "object"
|
||||
},
|
||||
"ROIBranchMetrics": {
|
||||
"properties": {
|
||||
"cost_per_hour": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Cost Per Hour"
|
||||
},
|
||||
"hours": {
|
||||
"default": 0,
|
||||
"title": "Hours",
|
||||
"type": "number"
|
||||
},
|
||||
"matched_pulls": {
|
||||
"default": 0,
|
||||
"title": "Matched Pulls",
|
||||
"type": "integer"
|
||||
},
|
||||
"spend": {
|
||||
"default": 0,
|
||||
"title": "Spend",
|
||||
"type": "number"
|
||||
},
|
||||
"total_tagged_spend": {
|
||||
"default": 0,
|
||||
"title": "Total Tagged Spend",
|
||||
"type": "number"
|
||||
},
|
||||
"unlinked_spend": {
|
||||
"default": 0,
|
||||
"title": "Unlinked Spend",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"title": "ROIBranchMetrics",
|
||||
"type": "object"
|
||||
},
|
||||
"ROIBranchSpend": {
|
||||
"properties": {
|
||||
"branch": {
|
||||
"title": "Branch",
|
||||
"type": "string"
|
||||
},
|
||||
"repo": {
|
||||
"title": "Repo",
|
||||
"type": "string"
|
||||
},
|
||||
"requests": {
|
||||
"title": "Requests",
|
||||
"type": "integer"
|
||||
},
|
||||
"spend": {
|
||||
"title": "Spend",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"repo",
|
||||
"branch",
|
||||
"spend",
|
||||
"requests"
|
||||
],
|
||||
"title": "ROIBranchSpend",
|
||||
"type": "object"
|
||||
},
|
||||
"ROIEstimateResponse": {
|
||||
"properties": {
|
||||
"cached": {
|
||||
|
|
@ -49261,6 +49376,9 @@
|
|||
"title": "Additions",
|
||||
"type": "integer"
|
||||
},
|
||||
"branch_cost": {
|
||||
"$ref": "#/components/schemas/ROIBranchAttribution"
|
||||
},
|
||||
"cache_key": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -49334,6 +49452,16 @@
|
|||
"title": "Repo",
|
||||
"type": "string"
|
||||
},
|
||||
"source_branch": {
|
||||
"default": "",
|
||||
"title": "Source Branch",
|
||||
"type": "string"
|
||||
},
|
||||
"source_repo": {
|
||||
"default": "",
|
||||
"title": "Source Repo",
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"title": "Title",
|
||||
"type": "string"
|
||||
|
|
@ -49463,6 +49591,11 @@
|
|||
"title": "Github Api Url",
|
||||
"type": "string"
|
||||
},
|
||||
"gitlab_api_url": {
|
||||
"default": "https://gitlab.com/api/v4",
|
||||
"title": "Gitlab Api Url",
|
||||
"type": "string"
|
||||
},
|
||||
"has_estimator_key": {
|
||||
"title": "Has Estimator Key",
|
||||
"type": "boolean"
|
||||
|
|
@ -49471,6 +49604,11 @@
|
|||
"title": "Has Github Token",
|
||||
"type": "boolean"
|
||||
},
|
||||
"has_gitlab_token": {
|
||||
"default": false,
|
||||
"title": "Has Gitlab Token",
|
||||
"type": "boolean"
|
||||
},
|
||||
"identity_map": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
|
|
@ -49489,6 +49627,15 @@
|
|||
"title": "Repos",
|
||||
"type": "array"
|
||||
},
|
||||
"source_provider": {
|
||||
"default": "github",
|
||||
"enum": [
|
||||
"github",
|
||||
"gitlab"
|
||||
],
|
||||
"title": "Source Provider",
|
||||
"type": "string"
|
||||
},
|
||||
"update_interval_minutes": {
|
||||
"title": "Update Interval Minutes",
|
||||
"type": "number"
|
||||
|
|
@ -49582,6 +49729,28 @@
|
|||
],
|
||||
"title": "Github Token"
|
||||
},
|
||||
"gitlab_api_url": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Gitlab Api Url"
|
||||
},
|
||||
"gitlab_token": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Gitlab Token"
|
||||
},
|
||||
"repos": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -49596,6 +49765,21 @@
|
|||
],
|
||||
"title": "Repos"
|
||||
},
|
||||
"source_provider": {
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"github",
|
||||
"gitlab"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Source Provider"
|
||||
},
|
||||
"update_interval_minutes": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -49615,6 +49799,9 @@
|
|||
},
|
||||
"ROISummaryResponse": {
|
||||
"properties": {
|
||||
"branch_metrics": {
|
||||
"$ref": "#/components/schemas/ROIBranchMetrics"
|
||||
},
|
||||
"effort_basis": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -49677,6 +49864,15 @@
|
|||
"title": "Repos",
|
||||
"type": "array"
|
||||
},
|
||||
"source_provider": {
|
||||
"default": "github",
|
||||
"enum": [
|
||||
"github",
|
||||
"gitlab"
|
||||
],
|
||||
"title": "Source Provider",
|
||||
"type": "string"
|
||||
},
|
||||
"start": {
|
||||
"title": "Start",
|
||||
"type": "string"
|
||||
|
|
@ -49692,6 +49888,14 @@
|
|||
"title": "Trend",
|
||||
"type": "array"
|
||||
},
|
||||
"unlinked_branches": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ROIBranchSpend"
|
||||
},
|
||||
"title": "Unlinked Branches",
|
||||
"type": "array"
|
||||
},
|
||||
"warnings": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
|
|
|
|||
|
|
@ -4214,6 +4214,7 @@ class SpendLogsMetadata(TypedDict):
|
|||
vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None
|
||||
routing_decision: StandardLoggingRoutingDecision | None
|
||||
internal_call_origin: InternalCallOrigin | None
|
||||
litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
|
||||
guardrail_information: list[StandardLoggingGuardrailInformation] | None
|
||||
eval_information: Any | None
|
||||
status: StandardLoggingPayloadStatus
|
||||
|
|
|
|||
|
|
@ -339,6 +339,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
|
|||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
"standard_logging_object",
|
||||
"litellm_roi_estimator",
|
||||
"proxy_server_request",
|
||||
"secret_fields",
|
||||
"_guardrail_pipelines",
|
||||
|
|
@ -2565,6 +2566,10 @@ async def add_litellm_data_to_request(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
data[_metadata_variable_name]["litellm_roi_estimator"] = (
|
||||
getattr(request.state, "litellm_roi_estimator", False) is True
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
|
||||
|
||||
# Team/Project credential overrides from model_config
|
||||
|
|
|
|||
|
|
@ -3,7 +3,12 @@ from datetime import date, datetime, timedelta, timezone
|
|||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal
|
||||
from typing import (
|
||||
Annotated,
|
||||
Final,
|
||||
Literal,
|
||||
cast, # noqa: TID251 # PrismaWrapper dynamically delegates database methods
|
||||
)
|
||||
|
||||
import httpx
|
||||
from apscheduler.schedulers.asyncio import ( # pyright: ignore[reportMissingTypeStubs] # no upstream stubs
|
||||
|
|
@ -11,6 +16,7 @@ from apscheduler.schedulers.asyncio import ( # pyright: ignore[reportMissingTyp
|
|||
)
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
|
||||
from starlette.types import Receive, Scope, Send
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -20,13 +26,22 @@ from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKey
|
|||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper
|
||||
from litellm.proxy.roi_calculator.analytics import normalize_email, summarize
|
||||
from litellm.proxy.roi_calculator.branch_spend import BranchSpendDatabase, read_branch_spend
|
||||
from litellm.proxy.roi_calculator.estimator import CompletionCaller, EstimatorModel
|
||||
from litellm.proxy.roi_calculator.github import GitHub, SourceError
|
||||
from litellm.proxy.roi_calculator.sync import SpendReader, SyncManager, read_spend, spend_prisma_client
|
||||
from litellm.proxy.roi_calculator.github import SourceError
|
||||
from litellm.proxy.roi_calculator.source import create_source
|
||||
from litellm.proxy.roi_calculator.sync import (
|
||||
BranchSpendReader,
|
||||
SpendReader,
|
||||
SyncManager,
|
||||
read_spend,
|
||||
spend_prisma_client,
|
||||
)
|
||||
from litellm.proxy.roi_calculator.sync_store import SyncStore
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.types.roi_calculator import (
|
||||
DEFAULT_PROMPT,
|
||||
ROIBranchSpend,
|
||||
ROICompletionRequest,
|
||||
ROIIdentityMapResponse,
|
||||
ROIIdentityMapUpdate,
|
||||
|
|
@ -40,6 +55,7 @@ from litellm.types.roi_calculator import (
|
|||
ROISpendRecord,
|
||||
ROISummaryResponse,
|
||||
ROISyncStatus,
|
||||
normalize_source_login,
|
||||
)
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
|
@ -52,6 +68,9 @@ _ROI_TAGS: Final[list[str | Enum]] = ["roi calculator"] # mutable-ok: FastAPI r
|
|||
class _StoredSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
source_provider: Literal["github", "gitlab"] = "github"
|
||||
gitlab_api_url: str = "https://gitlab.com/api/v4"
|
||||
gitlab_token: str = ""
|
||||
github_api_url: str = "https://api.github.com"
|
||||
github_token: str = ""
|
||||
estimator_key: str = ""
|
||||
|
|
@ -181,6 +200,11 @@ async def _load_settings(repository: ConfigRepository) -> ROISettings:
|
|||
token: Final = decrypt_value_helper(stored.github_token, _SETTINGS_KEY) if stored.github_token else ""
|
||||
try:
|
||||
return ROISettings(
|
||||
source_provider=stored.source_provider,
|
||||
gitlab_api_url=stored.gitlab_api_url,
|
||||
gitlab_token=SecretStr(decrypt_value_helper(stored.gitlab_token, _SETTINGS_KEY) or "")
|
||||
if stored.gitlab_token
|
||||
else SecretStr(""),
|
||||
github_api_url=stored.github_api_url,
|
||||
github_token=SecretStr(token or ""),
|
||||
estimator_key=SecretStr(decrypt_value_helper(stored.estimator_key, _SETTINGS_KEY) or "")
|
||||
|
|
@ -202,8 +226,12 @@ async def _save_settings(
|
|||
settings: ROISettings,
|
||||
encrypted_token: str,
|
||||
encrypted_estimator_key: str,
|
||||
encrypted_gitlab_token: str = "",
|
||||
) -> None:
|
||||
stored: Final = _StoredSettings(
|
||||
source_provider=settings.source_provider,
|
||||
gitlab_api_url=settings.gitlab_api_url,
|
||||
gitlab_token=encrypted_gitlab_token,
|
||||
github_api_url=settings.github_api_url,
|
||||
github_token=encrypted_token,
|
||||
estimator_key=encrypted_estimator_key,
|
||||
|
|
@ -217,19 +245,28 @@ async def _save_settings(
|
|||
await repository.set_param(_SETTINGS_KEY, stored.model_dump(mode="json"))
|
||||
|
||||
|
||||
async def _load_report(repository: ConfigRepository) -> ROIReport | None:
|
||||
async def _load_report(repository: ConfigRepository, settings: ROISettings) -> ROIReport | None:
|
||||
parameter: Final = await repository.get_param(_REPORT_KEY)
|
||||
if parameter is None:
|
||||
if parameter is None or parameter.param_value is None:
|
||||
return None
|
||||
try:
|
||||
return TypeAdapter(ROIReport).validate_python(parameter.param_value)
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(parameter.param_value)
|
||||
except ValidationError:
|
||||
raise HTTPException(status_code=500, detail="Stored ROI Calculator report is invalid.") from None
|
||||
if (
|
||||
report.get("source_provider", "github") != settings.source_provider
|
||||
or report.get("source_api_url", settings.github_api_url) != settings.source_api_url
|
||||
):
|
||||
return None
|
||||
return report
|
||||
|
||||
|
||||
def _public_settings(settings: ROISettings) -> ROISettingsResponse:
|
||||
models: Final = _router_models()
|
||||
return ROISettingsResponse(
|
||||
source_provider=settings.source_provider,
|
||||
gitlab_api_url=settings.gitlab_api_url,
|
||||
has_gitlab_token=bool(settings.gitlab_token.get_secret_value()),
|
||||
github_api_url=settings.github_api_url,
|
||||
repos=settings.repos,
|
||||
estimator_model=settings.estimator_model,
|
||||
|
|
@ -267,7 +304,14 @@ def _gateway_http_client() -> AsyncHTTPHandler:
|
|||
|
||||
@lru_cache(maxsize=1)
|
||||
def _gateway_transport(app: FastAPI) -> httpx.ASGITransport:
|
||||
return httpx.ASGITransport(app=app)
|
||||
async def estimator_request(scope: Scope, receive: Receive, send: Send) -> None:
|
||||
await app(
|
||||
{**scope, "state": {**scope.get("state", {}), "litellm_roi_estimator": True}},
|
||||
receive,
|
||||
send,
|
||||
)
|
||||
|
||||
return httpx.ASGITransport(app=estimator_request)
|
||||
|
||||
|
||||
def _completion_caller(settings: ROISettings) -> CompletionCaller:
|
||||
|
|
@ -317,6 +361,21 @@ def _spend_reader(repository: ConfigRepository) -> SpendReader:
|
|||
return get_spend
|
||||
|
||||
|
||||
def _branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> BranchSpendReader:
|
||||
async def get_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]:
|
||||
return await read_branch_spend(
|
||||
cast( # cast-ok: PrismaWrapper delegates methods dynamically
|
||||
BranchSpendDatabase, repository.prisma_client.db
|
||||
),
|
||||
start,
|
||||
end,
|
||||
repos,
|
||||
casefold_repo=settings.source_provider == "github",
|
||||
)
|
||||
|
||||
return get_spend
|
||||
|
||||
|
||||
@router.get(
|
||||
"/roi-calculator/settings",
|
||||
response_model=ROISettingsResponse,
|
||||
|
|
@ -343,8 +402,26 @@ async def update_roi_calculator_settings(
|
|||
current: Final = await _load_settings(repository)
|
||||
if "github_api_url" in patch.model_fields_set and patch.github_api_url is None:
|
||||
raise HTTPException(status_code=422, detail="GitHub API URL cannot be null.")
|
||||
if "gitlab_api_url" in patch.model_fields_set and patch.gitlab_api_url is None:
|
||||
raise HTTPException(status_code=422, detail="GitLab API URL cannot be null.")
|
||||
provider: Final = patch.source_provider or current.source_provider
|
||||
gitlab_url: Final = patch.gitlab_api_url if patch.gitlab_api_url is not None else current.gitlab_api_url
|
||||
gitlab_changed: Final = gitlab_url.rstrip("/") != current.gitlab_api_url.rstrip("/")
|
||||
gitlab_token: Final = (
|
||||
(patch.gitlab_token or "")
|
||||
if "gitlab_token" in patch.model_fields_set
|
||||
else ""
|
||||
if gitlab_changed
|
||||
else current.gitlab_token.get_secret_value()
|
||||
)
|
||||
encrypted_gitlab: Final = (
|
||||
TypeAdapter(str).validate_python(encrypt_value_helper(gitlab_token)) if gitlab_token else ""
|
||||
)
|
||||
github_api_url: Final = patch.github_api_url if patch.github_api_url is not None else current.github_api_url
|
||||
github_url_changed: Final = github_api_url.rstrip("/") != current.github_api_url.rstrip("/")
|
||||
source_changed: Final = provider != current.source_provider or (
|
||||
gitlab_changed if provider == "gitlab" else github_url_changed
|
||||
)
|
||||
token_was_supplied: Final = "github_token" in patch.model_fields_set
|
||||
plaintext_token, encrypted_token = (
|
||||
(
|
||||
|
|
@ -368,23 +445,28 @@ async def update_roi_calculator_settings(
|
|||
)
|
||||
try:
|
||||
settings: Final = ROISettings(
|
||||
source_provider=provider,
|
||||
gitlab_api_url=gitlab_url,
|
||||
gitlab_token=SecretStr(gitlab_token),
|
||||
github_api_url=github_api_url,
|
||||
github_token=SecretStr(plaintext_token),
|
||||
estimator_key=SecretStr(estimator_key),
|
||||
update_interval_minutes=patch.update_interval_minutes
|
||||
if patch.update_interval_minutes is not None
|
||||
else current.update_interval_minutes,
|
||||
repos=patch.repos if patch.repos is not None else current.repos,
|
||||
repos=patch.repos if patch.repos is not None else () if source_changed else current.repos,
|
||||
estimator_model=(patch.estimator_model if patch.estimator_model is not None else current.estimator_model),
|
||||
estimator_prompt=(
|
||||
patch.estimator_prompt if patch.estimator_prompt is not None else current.estimator_prompt
|
||||
),
|
||||
backfill_days=(patch.backfill_days if patch.backfill_days is not None else current.backfill_days),
|
||||
identity_map=current.identity_map,
|
||||
identity_map=MappingProxyType({}) if source_changed else current.identity_map,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=exc.errors(include_context=False)) from None
|
||||
await _save_settings(repository, settings, encrypted_token, encrypted_estimator_key)
|
||||
await _save_settings(repository, settings, encrypted_token, encrypted_estimator_key, encrypted_gitlab)
|
||||
if source_changed:
|
||||
await repository.set_param(_REPORT_KEY, None)
|
||||
return _public_settings(settings)
|
||||
|
||||
|
||||
|
|
@ -400,7 +482,7 @@ async def get_roi_calculator_repositories(
|
|||
query: Annotated[str, Query(max_length=200)] = "",
|
||||
page: Annotated[int, Query(ge=1, le=1000)] = 1,
|
||||
) -> ROIRepositoriesResponse:
|
||||
github: Final = GitHub(await _load_settings(repository), transport)
|
||||
github: Final = create_source(await _load_settings(repository), transport)
|
||||
try:
|
||||
repos, has_more = await github.repositories(query, page)
|
||||
except SourceError as exc:
|
||||
|
|
@ -428,7 +510,7 @@ async def get_roi_calculator_sync_status(
|
|||
) -> ROISyncStatus:
|
||||
status: Final = await SyncStore(repository.prisma_client).status() or manager.status
|
||||
settings: Final = await _load_settings(repository)
|
||||
report: Final = await _load_report(repository)
|
||||
report: Final = await _load_report(repository, settings)
|
||||
next_update: Final = _next_update(settings, status, report)
|
||||
return status.model_copy(update=MappingProxyType({"next_update": next_update.isoformat() if next_update else None}))
|
||||
|
||||
|
|
@ -448,7 +530,7 @@ async def start_roi_calculator_sync(
|
|||
settings: Final = await _load_settings(repository)
|
||||
public: Final = _public_settings(settings)
|
||||
if not public.ready:
|
||||
raise HTTPException(status_code=409, detail="Connect GitHub, select repositories, and choose a router model.")
|
||||
raise HTTPException(status_code=409, detail="Connect a source, select repositories, and choose a router model.")
|
||||
if not await manager.start(
|
||||
settings,
|
||||
repository,
|
||||
|
|
@ -457,6 +539,7 @@ async def start_roi_calculator_sync(
|
|||
transport,
|
||||
_router_estimator_models(settings.estimator_model),
|
||||
SyncStore(repository.prisma_client),
|
||||
branch_spend_reader=_branch_spend_reader(repository, settings),
|
||||
):
|
||||
raise HTTPException(status_code=409, detail="A sync is already running.")
|
||||
return manager.status
|
||||
|
|
@ -493,10 +576,10 @@ async def get_roi_calculator_report(
|
|||
|
||||
sample: Final = summarize(sample_report(datetime.now(timezone.utc)), MappingProxyType({}))
|
||||
return ROIReportResponse(report=ROISummaryResponse.model_validate(sample))
|
||||
report: Final = await _load_report(repository)
|
||||
settings: Final = await _load_settings(repository)
|
||||
report: Final = await _load_report(repository, settings)
|
||||
if report is None:
|
||||
return ROIReportResponse(report=None)
|
||||
settings: Final = await _load_settings(repository)
|
||||
summary: Final = summarize(report, settings.identity_map)
|
||||
return ROIReportResponse(report=ROISummaryResponse.model_validate(summary))
|
||||
|
||||
|
|
@ -514,15 +597,22 @@ async def update_roi_calculator_identity_map(
|
|||
login: Final = update.github_login.strip().casefold()
|
||||
current: Final = await _load_settings(repository)
|
||||
current_stored: Final = await _load_stored_settings(repository)
|
||||
try:
|
||||
normalize_source_login(login, current.source_provider)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from None
|
||||
new_email: Final = normalize_email(update.email)
|
||||
if not login or (update.email is not None and not new_email):
|
||||
raise HTTPException(status_code=422, detail="Enter a GitHub login and a valid email address.")
|
||||
raise HTTPException(status_code=422, detail="Enter a source-control username and a valid email address.")
|
||||
identity_map: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({key: value for key, value in current.identity_map.items() if key != login})
|
||||
if update.email is None
|
||||
else MappingProxyType({**current.identity_map, login: new_email})
|
||||
)
|
||||
settings: Final = ROISettings(
|
||||
source_provider=current.source_provider,
|
||||
gitlab_api_url=current.gitlab_api_url,
|
||||
gitlab_token=current.gitlab_token,
|
||||
github_api_url=current.github_api_url,
|
||||
github_token=current.github_token,
|
||||
estimator_key=current.estimator_key,
|
||||
|
|
@ -533,8 +623,10 @@ async def update_roi_calculator_identity_map(
|
|||
backfill_days=current.backfill_days,
|
||||
identity_map=identity_map,
|
||||
)
|
||||
await _save_settings(repository, settings, current_stored.github_token, current_stored.estimator_key)
|
||||
report: Final = await _load_report(repository)
|
||||
await _save_settings(
|
||||
repository, settings, current_stored.github_token, current_stored.estimator_key, current_stored.gitlab_token
|
||||
)
|
||||
report: Final = await _load_report(repository, settings)
|
||||
summary: Final = summarize(report, settings.identity_map) if report is not None else None
|
||||
return ROIIdentityMapResponse(
|
||||
report=ROISummaryResponse.model_validate(summary) if summary is not None else None,
|
||||
|
|
@ -581,7 +673,7 @@ async def run_scheduled_sync() -> None:
|
|||
return
|
||||
store: Final = SyncStore(prisma_client)
|
||||
status: Final = await store.status() or _SYNC_MANAGER.status
|
||||
report: Final = await _load_report(repository)
|
||||
report: Final = await _load_report(repository, settings)
|
||||
next_update: Final = _next_update(settings, status, report)
|
||||
if next_update is None or next_update > datetime.now(timezone.utc):
|
||||
return
|
||||
|
|
@ -593,6 +685,7 @@ async def run_scheduled_sync() -> None:
|
|||
estimator_models=_router_estimator_models(settings.estimator_model),
|
||||
coordinator=store,
|
||||
scheduled_interval=settings.update_interval_minutes,
|
||||
branch_spend_reader=_branch_spend_reader(repository, settings),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -607,7 +700,7 @@ async def test_roi_calculator_connections(
|
|||
if not public.ready:
|
||||
raise HTTPException(status_code=409, detail="Choose repositories and an available estimator model first.")
|
||||
await _test_estimator_access(settings)
|
||||
github: Final = GitHub(settings, transport)
|
||||
github: Final = create_source(settings, transport)
|
||||
try:
|
||||
await github.test_repositories(settings.repos)
|
||||
except SourceError as exc:
|
||||
|
|
@ -643,7 +736,7 @@ async def reset_roi_calculator_setup(
|
|||
current: Final = await _load_settings(repository)
|
||||
stored: Final = await _load_stored_settings(repository)
|
||||
settings: Final = current.model_copy(update=MappingProxyType({"repos": ()}))
|
||||
await _save_settings(repository, settings, stored.github_token, stored.estimator_key)
|
||||
await _save_settings(repository, settings, stored.github_token, stored.estimator_key, stored.gitlab_token)
|
||||
await store.clear_report()
|
||||
return _public_settings(settings)
|
||||
finally:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
from litellm.types.roi_calculator import (
|
||||
ROIBranchAttribution,
|
||||
ROIBranchMetrics,
|
||||
ROIPersonSummary,
|
||||
ROIPullRecord,
|
||||
ROIPullSummary,
|
||||
|
|
@ -53,8 +55,12 @@ def _pull_summary(
|
|||
address: str,
|
||||
method: str,
|
||||
observed: frozenset[str],
|
||||
branch_cost: ROIBranchAttribution,
|
||||
) -> ROIPullSummary:
|
||||
return ROIPullSummary(
|
||||
source_repo=pull.get("source_repo", ""),
|
||||
source_branch=pull.get("source_branch", ""),
|
||||
branch_cost=branch_cost,
|
||||
repo=pull["repo"],
|
||||
number=pull["number"],
|
||||
title=pull["title"],
|
||||
|
|
@ -119,6 +125,9 @@ def _summarize_person(
|
|||
|
||||
|
||||
def summarize(report: ROIReport, mappings: Mapping[str, str]) -> ROISummary:
|
||||
from litellm.proxy.roi_calculator.branch_spend import attribute_branches
|
||||
|
||||
branch_costs: Final = attribute_branches(report["pulls"], report.get("branch_spend"))
|
||||
complete_scope: Final = not report.get("unavailable_repos", ())
|
||||
observed: Final = frozenset(
|
||||
normalized for normalized in (normalize_email(row["email"]) for row in report["spend"]) if normalized
|
||||
|
|
@ -143,7 +152,8 @@ def summarize(report: ROIReport, mappings: Mapping[str, str]) -> ROISummary:
|
|||
for key in sorted(people_keys)
|
||||
)
|
||||
pull_summaries: Final = tuple(
|
||||
_pull_summary(pull, address, method, observed) for pull, address, method in matched_pulls
|
||||
_pull_summary(pull, address, method, observed, branch_costs[(pull["repo"], pull["number"])])
|
||||
for pull, address, method in matched_pulls
|
||||
)
|
||||
eligible_emails: Final = frozenset(person["email"] for person in people if person["eligible"])
|
||||
dates: Final = tuple(
|
||||
|
|
@ -197,7 +207,26 @@ def summarize(report: ROIReport, mappings: Mapping[str, str]) -> ROISummary:
|
|||
)
|
||||
summary_people: Final = tuple(sorted(people, key=lambda person: (-person["hours"], person["id"])))
|
||||
summary_pulls: Final = tuple(sorted(pull_summaries, key=lambda pull: pull["merged_at"], reverse=True))
|
||||
branch_cohort: Final = tuple(
|
||||
pull
|
||||
for pull in pull_summaries
|
||||
if pull["branch_cost"].status == "matched" and pull["estimate"]["status"] == "estimated"
|
||||
)
|
||||
branch_spend: Final = sum(pull["branch_cost"].spend or 0 for pull in branch_cohort)
|
||||
branch_hours: Final = sum(pull["estimate"]["hours"] or 0 for pull in branch_cohort)
|
||||
linked: Final = frozenset((pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in branch_cohort)
|
||||
unlinked: Final = tuple(row for row in report.get("branch_spend", ()) if (row.repo, row.branch) not in linked)
|
||||
return ROISummary(
|
||||
source_provider=report.get("source_provider", "github"),
|
||||
branch_metrics=ROIBranchMetrics(
|
||||
spend=branch_spend,
|
||||
hours=branch_hours,
|
||||
cost_per_hour=branch_spend / branch_hours if complete_scope and branch_hours else None,
|
||||
matched_pulls=sum(pull["branch_cost"].status == "matched" for pull in pull_summaries),
|
||||
total_tagged_spend=sum(row.spend for row in report.get("branch_spend", ())),
|
||||
unlinked_spend=sum(row.spend for row in unlinked),
|
||||
),
|
||||
unlinked_branches=unlinked,
|
||||
id=report.get("id"),
|
||||
mode=report["mode"],
|
||||
start=report["start"],
|
||||
|
|
|
|||
81
litellm/proxy/roi_calculator/branch_spend.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import json
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.roi_calculator import ROIBranchAttribution, ROIBranchSpend, ROIPullRecord
|
||||
|
||||
|
||||
class BranchSpendDatabase(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> object: ...
|
||||
|
||||
|
||||
async def read_branch_spend(
|
||||
database: BranchSpendDatabase, start: date, end: date, repos: tuple[str, ...], *, casefold_repo: bool = False
|
||||
) -> tuple[ROIBranchSpend, ...]:
|
||||
if not repos:
|
||||
return ()
|
||||
query: Final = """
|
||||
WITH tagged AS (
|
||||
SELECT logs.spend, tags.repos[1] AS repo, tags.branches[1] AS branch
|
||||
FROM "LiteLLM_SpendLogs" AS logs
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT array_agg(DISTINCT substring(tag FROM 6))
|
||||
FILTER (WHERE starts_with(tag, 'repo:')) AS repos,
|
||||
array_agg(DISTINCT substring(tag FROM 8))
|
||||
FILTER (WHERE starts_with(tag, 'branch:')) AS branches
|
||||
FROM jsonb_array_elements_text(
|
||||
CASE WHEN jsonb_typeof(logs.request_tags) = 'array'
|
||||
THEN logs.request_tags ELSE '[]'::jsonb END
|
||||
) AS tag
|
||||
) AS tags
|
||||
WHERE logs."startTime" >= $1::text::timestamp AND logs."startTime" < $2::text::timestamp
|
||||
AND cardinality(tags.repos) = 1 AND cardinality(tags.branches) = 1
|
||||
AND CASE logs.metadata -> 'litellm_roi_estimator'
|
||||
WHEN 'true'::jsonb THEN false
|
||||
WHEN 'false'::jsonb THEN true
|
||||
ELSE NOT coalesce(logs.request_tags ? 'litellm-roi-estimator', false)
|
||||
END
|
||||
)
|
||||
SELECT CASE WHEN $4 THEN lower(repo) ELSE repo END AS repo,
|
||||
branch, sum(spend)::double precision AS spend, count(*)::integer AS requests
|
||||
FROM tagged
|
||||
WHERE branch <> '' AND (CASE WHEN $4 THEN lower(repo) ELSE repo END)
|
||||
IN (SELECT jsonb_array_elements_text($3::jsonb))
|
||||
GROUP BY 1, 2
|
||||
ORDER BY 1, 2
|
||||
"""
|
||||
result: Final = await database.query_raw(
|
||||
query,
|
||||
datetime.combine(start, time.min, timezone.utc).isoformat(),
|
||||
datetime.combine(end + timedelta(days=1), time.min, timezone.utc).isoformat(),
|
||||
json.dumps(repos),
|
||||
casefold_repo,
|
||||
)
|
||||
return TypeAdapter(tuple[ROIBranchSpend, ...]).validate_python(result)
|
||||
|
||||
|
||||
def attribute_branches(
|
||||
pulls: tuple[ROIPullRecord, ...], spend: tuple[ROIBranchSpend, ...] | None
|
||||
) -> Mapping[tuple[str, int], ROIBranchAttribution]:
|
||||
counts: Final = Counter((pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in pulls)
|
||||
costs: Final = {(row.repo, row.branch): row for row in spend or ()}
|
||||
|
||||
def attribute(pull: ROIPullRecord) -> ROIBranchAttribution:
|
||||
repo: Final = pull.get("source_repo", "")
|
||||
branch: Final = pull.get("source_branch", "")
|
||||
cost: Final = costs.get((repo, branch))
|
||||
if spend is None:
|
||||
return ROIBranchAttribution(repo=repo, branch=branch, status="unavailable")
|
||||
if not repo or not branch or cost is None:
|
||||
return ROIBranchAttribution(repo=repo, branch=branch)
|
||||
if counts[(repo, branch)] != 1:
|
||||
return ROIBranchAttribution(repo=repo, branch=branch, status="ambiguous")
|
||||
return ROIBranchAttribution(
|
||||
repo=repo, branch=branch, spend=cost.spend, requests=cost.requests, status="matched"
|
||||
)
|
||||
|
||||
return {(pull["repo"], pull["number"]): attribute(pull) for pull in pulls}
|
||||
|
|
@ -112,14 +112,16 @@ class Estimator:
|
|||
missing_metadata_estimate: Final[ROIEstimate] = {
|
||||
"status": "needs_review",
|
||||
"hours": None,
|
||||
"reasoning": ("GitHub did not provide all file or commit metadata. It was not sent for estimation."),
|
||||
"reasoning": (
|
||||
"The repository source did not provide all file or commit metadata. It was not sent for estimation."
|
||||
),
|
||||
}
|
||||
return missing_metadata_estimate
|
||||
if len(evidence) > MAX_EVIDENCE_CHARS:
|
||||
oversized_evidence_estimate: Final[ROIEstimate] = {
|
||||
"status": "needs_review",
|
||||
"hours": None,
|
||||
"reasoning": ("This PR exceeds the estimator's input limit. It was not truncated or scored."),
|
||||
"reasoning": ("This change exceeds the estimator's input limit. It was not truncated or scored."),
|
||||
}
|
||||
return oversized_evidence_estimate
|
||||
system_message: Final[ROICompletionMessage] = {
|
||||
|
|
@ -131,7 +133,6 @@ class Estimator:
|
|||
response_format: Final[ROIResponseFormat] = {"type": "json_object"}
|
||||
metadata: Final[ROICompletionMetadata] = {
|
||||
"tags": ("litellm-roi-estimator",),
|
||||
"litellm_roi_estimator": True,
|
||||
}
|
||||
request: Final = ROICompletionRequest(
|
||||
model=self.settings.estimator_model,
|
||||
|
|
|
|||
|
|
@ -31,8 +31,14 @@ class _GitHubUser(_GitHubModel):
|
|||
login: str | None = None
|
||||
|
||||
|
||||
class _GitHubHeadRepository(_GitHubModel):
|
||||
full_name: str = ""
|
||||
|
||||
|
||||
class _GitHubHead(_GitHubModel):
|
||||
sha: str = ""
|
||||
ref: str = ""
|
||||
repo: _GitHubHeadRepository | None = None
|
||||
|
||||
|
||||
class GitHubPullListItem(_GitHubModel):
|
||||
|
|
@ -315,6 +321,7 @@ class GitHub:
|
|||
) -> None:
|
||||
if client is not None and transport is not None:
|
||||
raise ValueError("Pass either an injected GitHub client or a transport.")
|
||||
self._settings: Final = settings
|
||||
self._profiles: Mapping[str, str | None] = MappingProxyType({})
|
||||
token: Final = settings.github_token.get_secret_value()
|
||||
self._headers: Final[Mapping[str, str]] = (
|
||||
|
|
@ -472,7 +479,11 @@ class GitHub:
|
|||
if address
|
||||
)
|
||||
changed_files: Final = detail.changed_files if detail.changed_files is not None else len(files)
|
||||
from litellm.proxy.roi_calculator.source import repository_tag
|
||||
|
||||
evidence: Final[ROIPullEvidence] = {
|
||||
"source_repo": repository_tag(self._settings, detail.head.repo.full_name) if detail.head.repo else "",
|
||||
"source_branch": detail.head.ref,
|
||||
"repo": repo,
|
||||
"number": detail.number,
|
||||
"title": detail.title,
|
||||
|
|
|
|||
267
litellm/proxy/roi_calculator/gitlab.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from datetime import date
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params
|
||||
)
|
||||
from litellm.proxy.roi_calculator.analytics import normalize_email
|
||||
from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError
|
||||
from litellm.proxy.roi_calculator.source import repository_tag
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
from litellm.types.roi_calculator import ROIPullCommit, ROIPullEvidence, ROIPullFile, ROISettings
|
||||
|
||||
_T: Final = TypeVar("_T", bound=BaseModel)
|
||||
|
||||
|
||||
class _User(BaseModel):
|
||||
username: str
|
||||
public_email: str | None = None
|
||||
|
||||
|
||||
class _Project(BaseModel):
|
||||
id: int
|
||||
path_with_namespace: str
|
||||
visibility: str = "private"
|
||||
archived: bool = False
|
||||
|
||||
|
||||
class _MergeRequest(BaseModel):
|
||||
iid: int
|
||||
title: str
|
||||
description: str | None = None
|
||||
web_url: str
|
||||
author: _User
|
||||
merged_at: str | None
|
||||
updated_at: str
|
||||
sha: str | None = None
|
||||
source_branch: str
|
||||
source_project_id: int | None
|
||||
changes_count: str | None = None
|
||||
|
||||
def pull(self, source: _Project | None) -> GitHubPullListItem:
|
||||
return GitHubPullListItem.model_validate(
|
||||
{
|
||||
"number": self.iid,
|
||||
"title": self.title,
|
||||
"body": self.description or "",
|
||||
"html_url": self.web_url,
|
||||
"user": {"login": self.author.username},
|
||||
"merged_at": self.merged_at,
|
||||
"updated_at": self.updated_at,
|
||||
"head": {
|
||||
"sha": self.sha or "",
|
||||
"ref": self.source_branch,
|
||||
"repo": {"full_name": source.path_with_namespace} if source else None,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _Diff(BaseModel):
|
||||
new_path: str
|
||||
old_path: str
|
||||
diff: str = ""
|
||||
new_file: bool = False
|
||||
deleted_file: bool = False
|
||||
renamed_file: bool = False
|
||||
collapsed: bool = False
|
||||
too_large: bool = False
|
||||
|
||||
def file(self) -> ROIPullFile:
|
||||
return ROIPullFile(
|
||||
filename=self.new_path,
|
||||
status="added"
|
||||
if self.new_file
|
||||
else "removed"
|
||||
if self.deleted_file
|
||||
else "renamed"
|
||||
if self.renamed_file
|
||||
else "modified",
|
||||
additions=sum(line.startswith("+") for line in self.diff.splitlines()),
|
||||
deletions=sum(line.startswith("-") for line in self.diff.splitlines()),
|
||||
)
|
||||
|
||||
|
||||
class _Commit(BaseModel):
|
||||
id: str
|
||||
message: str
|
||||
|
||||
|
||||
class GitLab:
|
||||
def __init__(self, settings: ROISettings, transport: httpx.AsyncBaseTransport | None = None) -> None:
|
||||
self.settings: Final = settings
|
||||
token: Final = settings.gitlab_token.get_secret_value()
|
||||
self.headers: Final = {"Accept": "application/json", **({"PRIVATE-TOKEN": token} if token else {})}
|
||||
self.client: Final = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.ROICalculator,
|
||||
params={"timeout": 45, "follow_redirects": False, "transport": transport},
|
||||
).client
|
||||
self.close_client: Final = transport is not None
|
||||
self.profiles: Mapping[str, str] = MappingProxyType({})
|
||||
self.projects: Mapping[int, _Project] = MappingProxyType({})
|
||||
self.source_project_slots: Final = asyncio.Semaphore(8)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.close_client:
|
||||
await self.client.aclose()
|
||||
|
||||
async def _request(
|
||||
self, path: str, params: Mapping[str, str | int] | None = None, attempt: int = 0
|
||||
) -> httpx.Response:
|
||||
try:
|
||||
response: Final = await self.client.get(
|
||||
self.settings.gitlab_api_url + "/" + path, params=params, headers=self.headers
|
||||
)
|
||||
except httpx.RequestError:
|
||||
raise SourceError("Could not reach GitLab. Check the API URL and network connection.") from None
|
||||
if response.status_code in (429, 502, 503, 504) and attempt < 2:
|
||||
await asyncio.sleep(0.5 * (attempt + 1))
|
||||
return await self._request(path, params, attempt + 1)
|
||||
if response.status_code != 200:
|
||||
raise SourceError(
|
||||
f"GitLab could not read this resource (HTTP {response.status_code}). "
|
||||
"Check the project, token read_api scope, and project membership."
|
||||
)
|
||||
return response
|
||||
|
||||
async def _page(
|
||||
self, path: str, model: type[_T], params: Mapping[str, str | int], page: int
|
||||
) -> tuple[tuple[_T, ...], bool]:
|
||||
response: Final = await self._request(path, {**params, "per_page": 100, "page": page})
|
||||
try:
|
||||
values: Final = TypeAdapter(tuple[object, ...]).validate_python(response.json())
|
||||
items: Final = tuple(model.model_validate(value) for value in values)
|
||||
except ValueError:
|
||||
raise SourceError("GitLab returned an invalid page of results.") from None
|
||||
has_more: Final = response.headers.get("x-next-page", "") != "" or 'rel="next"' in response.headers.get(
|
||||
"link", ""
|
||||
)
|
||||
return items, has_more
|
||||
|
||||
async def _all(self, path: str, model: type[_T], params: Mapping[str, str | int] | None = None) -> tuple[_T, ...]:
|
||||
async def collect(page: int, previous: tuple[_T, ...]) -> tuple[_T, ...]:
|
||||
items, more = await self._page(path, model, params or {}, page)
|
||||
if not more:
|
||||
return previous + items
|
||||
if page >= 100:
|
||||
raise SourceError("GitLab's pagination limit was reached. Narrow the reporting window.")
|
||||
return await collect(page + 1, previous + items)
|
||||
|
||||
return await collect(1, ())
|
||||
|
||||
async def _project(self, project: str | int) -> _Project:
|
||||
if isinstance(project, int) and project in self.projects:
|
||||
return self.projects[project]
|
||||
response: Final = await self._request("projects/" + quote(str(project), safe=""))
|
||||
try:
|
||||
result: Final = _Project.model_validate(response.json())
|
||||
except ValueError:
|
||||
raise SourceError("GitLab returned invalid project details.") from None
|
||||
self.projects = MappingProxyType({**self.projects, result.id: result})
|
||||
return result
|
||||
|
||||
async def repositories(self, query: str = "", page: int = 1) -> tuple[tuple[tuple[str, str, bool], ...], bool]:
|
||||
params: Final = {
|
||||
"simple": "true",
|
||||
"search": query,
|
||||
**({"membership": "true"} if self.headers.get("PRIVATE-TOKEN") else {}),
|
||||
}
|
||||
items, more = await self._page("projects", _Project, params, page)
|
||||
return tuple((item.path_with_namespace, item.visibility, item.archived) for item in items), more
|
||||
|
||||
async def test_repositories(self, repos: tuple[str, ...]) -> None:
|
||||
async def test(repo: str) -> None:
|
||||
project: Final = await self._project(repo)
|
||||
await self._request(f"projects/{project.id}/merge_requests", {"state": "merged", "per_page": 1})
|
||||
|
||||
for repo in repos:
|
||||
await test(repo)
|
||||
|
||||
async def pulls(self, repo: str, start: date, end: date) -> tuple[GitHubPullListItem, ...]:
|
||||
project: Final = await self._project(repo)
|
||||
items: Final = await self._all(
|
||||
f"projects/{project.id}/merge_requests",
|
||||
_MergeRequest,
|
||||
{
|
||||
"state": "merged",
|
||||
"scope": "all",
|
||||
"updated_after": start.isoformat() + "T00:00:00Z",
|
||||
"order_by": "updated_at",
|
||||
"sort": "desc",
|
||||
},
|
||||
)
|
||||
merged: Final = tuple(
|
||||
item for item in items if item.merged_at and start.isoformat() <= item.merged_at[:10] <= end.isoformat()
|
||||
)
|
||||
source_ids: Final = tuple(frozenset(item.source_project_id for item in merged))
|
||||
projects: Final = await asyncio.gather(*(self._source_project(source_id) for source_id in source_ids))
|
||||
sources: Final = MappingProxyType(dict(zip(source_ids, projects, strict=True)))
|
||||
return tuple(item.pull(sources[item.source_project_id]) for item in merged)
|
||||
|
||||
async def profile_email(self, login: str, *, fallback: str = "") -> str:
|
||||
if login.casefold() in self.profiles:
|
||||
return self.profiles[login.casefold()]
|
||||
try:
|
||||
users: Final = await self._all("users", _User, {"username": login})
|
||||
except SourceError:
|
||||
return fallback
|
||||
email: Final = next(
|
||||
(normalize_email(user.public_email) for user in users if user.username.casefold() == login.casefold()), ""
|
||||
)
|
||||
self.profiles = MappingProxyType({**self.profiles, login.casefold(): email})
|
||||
return email
|
||||
|
||||
async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence:
|
||||
project: Final = await self._project(repo)
|
||||
path: Final = f"projects/{project.id}/merge_requests/{pull.number}"
|
||||
response: Final = await self._request(path)
|
||||
try:
|
||||
detail: Final = _MergeRequest.model_validate(response.json())
|
||||
except ValueError:
|
||||
raise SourceError("GitLab returned invalid merge request details.") from None
|
||||
diffs: Final = await self._all(path + "/diffs", _Diff)
|
||||
commits: Final = await self._all(path + "/commits", _Commit)
|
||||
profile: Final = await self.profile_email(detail.author.username)
|
||||
source: Final = await self._source_project(detail.source_project_id)
|
||||
files: Final = tuple(diff.file() for diff in diffs)
|
||||
return ROIPullEvidence(
|
||||
repo=repo,
|
||||
number=detail.iid,
|
||||
title=detail.title,
|
||||
body=detail.description or "",
|
||||
url=detail.web_url,
|
||||
login=detail.author.username,
|
||||
emails=(profile,) if profile else (),
|
||||
profile_email=profile,
|
||||
commit_emails=(),
|
||||
merged_at=detail.merged_at or "",
|
||||
head_sha=detail.sha or "",
|
||||
source_repo=repository_tag(self.settings, source.path_with_namespace) if source else "",
|
||||
source_branch=detail.source_branch,
|
||||
additions=sum(file["additions"] or 0 for file in files),
|
||||
deletions=sum(file["deletions"] or 0 for file in files),
|
||||
changed_files=len(files),
|
||||
files=files,
|
||||
commits=tuple(ROIPullCommit(sha=commit.id, message=commit.message) for commit in commits),
|
||||
commit_count=len(commits),
|
||||
incomplete_metadata=any(diff.collapsed or diff.too_large for diff in diffs)
|
||||
or detail.changes_count is None
|
||||
or not detail.changes_count.isdigit()
|
||||
or int(detail.changes_count) != len(files),
|
||||
)
|
||||
|
||||
async def _source_project(self, project_id: int | None) -> _Project | None:
|
||||
if project_id is None:
|
||||
return None
|
||||
try:
|
||||
async with self.source_project_slots:
|
||||
return await self._project(project_id)
|
||||
except SourceError:
|
||||
return None
|
||||
|
|
@ -18,12 +18,15 @@ def cache_key(
|
|||
return None
|
||||
value: Final = json.dumps(
|
||||
(
|
||||
"pull-v1",
|
||||
settings.github_api_url.rstrip("/"),
|
||||
"pull-v2-branches",
|
||||
settings.source_provider,
|
||||
settings.source_api_url.rstrip("/"),
|
||||
context,
|
||||
repo.casefold(),
|
||||
repo.casefold() if settings.source_provider == "github" else repo,
|
||||
pull.number,
|
||||
head,
|
||||
pull.head.ref if pull.head is not None else "",
|
||||
pull.head.repo.full_name if pull.head is not None and pull.head.repo is not None else "",
|
||||
pull.title,
|
||||
pull.body or "",
|
||||
login.casefold(),
|
||||
|
|
@ -36,7 +39,8 @@ def cache_key(
|
|||
def settings_fingerprint(settings: ROISettings) -> str:
|
||||
value: Final = json.dumps(
|
||||
(
|
||||
settings.github_api_url.rstrip("/"),
|
||||
settings.source_provider,
|
||||
settings.source_api_url.rstrip("/"),
|
||||
settings.repos,
|
||||
settings.estimator_model,
|
||||
settings.estimator_prompt,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,14 @@
|
|||
from datetime import datetime, timedelta
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.roi_calculator import DEFAULT_PROMPT, ROIEstimate, ROIPullRecord, ROIReport, ROISpendRecord
|
||||
from litellm.types.roi_calculator import (
|
||||
DEFAULT_PROMPT,
|
||||
ROIBranchSpend,
|
||||
ROIEstimate,
|
||||
ROIPullRecord,
|
||||
ROIReport,
|
||||
ROISpendRecord,
|
||||
)
|
||||
|
||||
|
||||
def sample_report(now: datetime) -> ROIReport:
|
||||
|
|
@ -9,8 +16,10 @@ def sample_report(now: datetime) -> ROIReport:
|
|||
examples: Final = (
|
||||
("alex", "alex@example.com", "Add usage breakdown by model", 6.5, 18.2),
|
||||
("jordan", "jordan@example.com", "Fix streaming response cancellation", 4.0, 12.8),
|
||||
("casey", "", "Add integration tests for billing", 5.5, 0.0),
|
||||
("casey", "", "Add integration tests for billing", 5.5, 7.4),
|
||||
)
|
||||
branches: Final = ("feature/model-usage", "fix/stream-cancellation", "test/billing-integration")
|
||||
branch_costs: Final = (9.1, 6.4, 7.4)
|
||||
|
||||
def pull(index: int, login: str, email: str, title: str, hours: float) -> ROIPullRecord:
|
||||
estimate: Final[ROIEstimate] = {
|
||||
|
|
@ -23,6 +32,8 @@ def sample_report(now: datetime) -> ROIReport:
|
|||
"cached": False,
|
||||
}
|
||||
return ROIPullRecord(
|
||||
source_repo="github.com/example/gateway",
|
||||
source_branch=branches[index],
|
||||
repo="example/gateway",
|
||||
number=142 + index,
|
||||
title=title,
|
||||
|
|
@ -47,9 +58,13 @@ def sample_report(now: datetime) -> ROIReport:
|
|||
spend: Final = tuple(
|
||||
ROISpendRecord(date=pulls[index]["merged_at"][:10], user_id=login, email=email, spend=cost, requests=150)
|
||||
for index, (login, email, _, _, cost) in enumerate(examples)
|
||||
if email
|
||||
)
|
||||
return ROIReport(
|
||||
branch_spend=tuple(
|
||||
ROIBranchSpend(repo="github.com/example/gateway", branch=branch, spend=cost, requests=75)
|
||||
for branch, cost in zip(branches, branch_costs)
|
||||
)
|
||||
+ (ROIBranchSpend(repo="github.com/example/gateway", branch="feature/cost-export", spend=3.6, requests=30),),
|
||||
mode="demo",
|
||||
start=start.isoformat(),
|
||||
end=now.date().isoformat(),
|
||||
|
|
|
|||
33
litellm/proxy/roi_calculator/source.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from datetime import date
|
||||
from typing import Final, Protocol
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.proxy.roi_calculator.github import GitHub, GitHubPullListItem
|
||||
from litellm.types.roi_calculator import ROIPullEvidence, ROISettings
|
||||
|
||||
|
||||
class RepositorySource(Protocol):
|
||||
async def repositories(self, query: str = "", page: int = 1) -> tuple[tuple[tuple[str, str, bool], ...], bool]: ...
|
||||
async def test_repositories(self, repos: tuple[str, ...]) -> None: ...
|
||||
async def pulls(self, repo: str, start: date, end: date) -> tuple[GitHubPullListItem, ...]: ...
|
||||
async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence: ...
|
||||
async def profile_email(self, login: str, *, fallback: str = "") -> str: ...
|
||||
async def close(self) -> None: ...
|
||||
|
||||
|
||||
def repository_tag(settings: ROISettings, repo: str) -> str:
|
||||
parsed: Final = urlsplit(settings.source_api_url)
|
||||
host: Final = "github.com" if parsed.netloc == "api.github.com" else parsed.netloc
|
||||
prefix: Final = parsed.path.removesuffix("/api/v4").removesuffix("/api/v3").rstrip("/")
|
||||
value: Final = host + prefix + "/" + repo
|
||||
return value.casefold() if settings.source_provider == "github" else value
|
||||
|
||||
|
||||
def create_source(settings: ROISettings, transport: httpx.AsyncBaseTransport | None = None) -> RepositorySource:
|
||||
if settings.source_provider == "gitlab":
|
||||
from litellm.proxy.roi_calculator.gitlab import GitLab
|
||||
|
||||
return GitLab(settings, transport)
|
||||
return GitHub(settings, transport)
|
||||
|
|
@ -11,11 +11,14 @@ import httpx
|
|||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict, Unpack
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.roi_calculator.estimator import CompletionCaller, Estimator, EstimatorModel, cache_context
|
||||
from litellm.proxy.roi_calculator.github import GitHub, GitHubPullListItem, SourceError
|
||||
from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError
|
||||
from litellm.proxy.roi_calculator.pull_cache import cache_key, settings_fingerprint
|
||||
from litellm.proxy.roi_calculator.source import RepositorySource, create_source, repository_tag
|
||||
from litellm.repositories.chunked_in import find_many_in
|
||||
from litellm.types.roi_calculator import (
|
||||
ROIBranchSpend,
|
||||
ROIEstimate,
|
||||
ROIPullEvidence,
|
||||
ROIPullRecord,
|
||||
|
|
@ -31,6 +34,10 @@ _REPORT_ADAPTER: Final = TypeAdapter(ROIReport)
|
|||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class _BranchSpendFields(TypedDict, total=False):
|
||||
branch_spend: ReadOnly[tuple[ROIBranchSpend, ...]]
|
||||
|
||||
|
||||
class _ConfigParam(Protocol):
|
||||
@property
|
||||
def param_value(self) -> object: ...
|
||||
|
|
@ -164,7 +171,7 @@ class GitHubFactory(Protocol):
|
|||
self,
|
||||
settings: ROISettings,
|
||||
transport: httpx.AsyncBaseTransport | None,
|
||||
) -> GitHub: ...
|
||||
) -> RepositorySource: ...
|
||||
|
||||
|
||||
class SpendReader(Protocol):
|
||||
|
|
@ -175,6 +182,10 @@ class SpendReader(Protocol):
|
|||
) -> Awaitable[tuple[ROISpendRecord, ...]]: ...
|
||||
|
||||
|
||||
class BranchSpendReader(Protocol):
|
||||
def __call__(self, start: date, end: date, repos: tuple[str, ...]) -> Awaitable[tuple[ROIBranchSpend, ...]]: ...
|
||||
|
||||
|
||||
class SyncClock(Protocol):
|
||||
def __call__(self) -> datetime: ...
|
||||
|
||||
|
|
@ -210,7 +221,9 @@ async def _estimate_with_fallback(
|
|||
return estimate
|
||||
|
||||
|
||||
async def _unavailable_record(github: GitHub, repo: str, pull: GitHubPullListItem, error: SourceError) -> ROIPullRecord:
|
||||
async def _unavailable_record(
|
||||
github: RepositorySource, settings: ROISettings, repo: str, pull: GitHubPullListItem, error: SourceError
|
||||
) -> ROIPullRecord:
|
||||
login: Final = pull.user.login if pull.user and pull.user.login else "deleted-user"
|
||||
profile: Final = await github.profile_email(login)
|
||||
estimate: Final[ROIEstimate] = {
|
||||
|
|
@ -219,6 +232,10 @@ async def _unavailable_record(github: GitHub, repo: str, pull: GitHubPullListIte
|
|||
"reasoning": f"PR metadata could not be read: {error} Run analysis again to retry this PR.",
|
||||
}
|
||||
return ROIPullRecord(
|
||||
source_repo=repository_tag(settings, pull.head.repo.full_name)
|
||||
if pull.head and pull.head.repo and pull.head.repo.full_name
|
||||
else "",
|
||||
source_branch=pull.head.ref if pull.head else "",
|
||||
repo=repo,
|
||||
number=pull.number,
|
||||
title=pull.title,
|
||||
|
|
@ -258,25 +275,27 @@ class _RepositoryBatch(NamedTuple):
|
|||
stage: str
|
||||
|
||||
|
||||
async def _read_repository(github: GitHub, repo: str, start: date, end: date) -> _RepositoryPulls:
|
||||
async def _read_repository(github: RepositorySource, repo: str, start: date, end: date) -> _RepositoryPulls:
|
||||
try:
|
||||
return _RepositoryPulls(repo, await github.pulls(repo, start, end))
|
||||
except SourceError:
|
||||
return _RepositoryPulls(repo, (), unavailable=True)
|
||||
|
||||
|
||||
async def _read_repositories(github: GitHub, repos: tuple[str, ...], start: date, end: date) -> _RepositoryBatch:
|
||||
async def _read_repositories(
|
||||
github: RepositorySource, repos: tuple[str, ...], start: date, end: date
|
||||
) -> _RepositoryBatch:
|
||||
groups: Final = await asyncio.gather(*(_read_repository(github, repo, start, end) for repo in repos))
|
||||
unavailable: Final = tuple(group.repo for group in groups if group.unavailable)
|
||||
if len(unavailable) == len(repos):
|
||||
raise SourceError(
|
||||
"GitHub could not read any selected repository. No new report was published; "
|
||||
"The repository source could not read any selected repository. No new report was published; "
|
||||
"check repository access or try analysis again later."
|
||||
)
|
||||
queue: Final = tuple(chain.from_iterable(((group.repo, pull) for pull in group.pulls) for group in groups))
|
||||
if unavailable and not queue:
|
||||
raise SourceError(
|
||||
f"GitHub could not read {', '.join(unavailable)}, and the accessible repositories returned no pull requests. "
|
||||
f"The repository source could not read {', '.join(unavailable)}, and the accessible repositories returned no merged changes. "
|
||||
"No new report was published; check repository access or try analysis again later."
|
||||
)
|
||||
warnings: Final = (
|
||||
|
|
@ -301,13 +320,13 @@ async def _read_repositories(github: GitHub, repos: tuple[str, ...], start: date
|
|||
def _processed_records(processed: tuple[_ProcessedPull, ...]) -> Mapping[int, ROIPullRecord]:
|
||||
if processed and all(item.metadata_unavailable for item in processed):
|
||||
raise SourceError(
|
||||
"GitHub could not provide PR metadata. No new report was published; try analysis again later."
|
||||
"The repository source could not provide PR metadata. No new report was published; try analysis again later."
|
||||
)
|
||||
if any(item.record["estimate"]["status"] == "error" for item in processed) and not any(
|
||||
item.record["estimate"]["status"] == "estimated" for item in processed
|
||||
):
|
||||
raise SourceError(
|
||||
"The estimator could not score any pull requests. No new report was published; "
|
||||
"The estimator could not score any merged changes. No new report was published; "
|
||||
"check the estimator connection or try analysis again later."
|
||||
)
|
||||
return MappingProxyType({item.position: item.record for item in processed})
|
||||
|
|
@ -332,7 +351,7 @@ async def _cache_estimated_pull(
|
|||
class SyncManager:
|
||||
def __init__(
|
||||
self,
|
||||
github_factory: GitHubFactory = GitHub,
|
||||
github_factory: GitHubFactory = create_source,
|
||||
clock: SyncClock = _utc_now,
|
||||
) -> None:
|
||||
self._github_factory: Final = github_factory
|
||||
|
|
@ -379,6 +398,7 @@ class SyncManager:
|
|||
estimator_models: tuple[EstimatorModel, ...] | None = None,
|
||||
coordinator: SyncCoordinator | None = None,
|
||||
scheduled_interval: float = 0,
|
||||
branch_spend_reader: BranchSpendReader | None = None,
|
||||
) -> bool:
|
||||
async with self._start_lock:
|
||||
if not settings.repos or not settings.estimator_model:
|
||||
|
|
@ -410,7 +430,15 @@ class SyncManager:
|
|||
self._owner = owner
|
||||
self._task = asyncio.create_task(
|
||||
self._run(
|
||||
settings, repository, spend_reader, complete, github_transport, estimator_models, coordinator, owner
|
||||
settings,
|
||||
repository,
|
||||
spend_reader,
|
||||
complete,
|
||||
github_transport,
|
||||
estimator_models,
|
||||
coordinator,
|
||||
owner,
|
||||
branch_spend_reader,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
|
@ -452,6 +480,7 @@ class SyncManager:
|
|||
estimator_models: tuple[EstimatorModel, ...] | None,
|
||||
coordinator: SyncCoordinator | None,
|
||||
owner: str,
|
||||
branch_spend_reader: BranchSpendReader | None,
|
||||
) -> None:
|
||||
monitor: Final = asyncio.create_task(self._heartbeat(asyncio.current_task(), coordinator, owner))
|
||||
github: Final = self._github_factory(settings, github_transport)
|
||||
|
|
@ -477,7 +506,7 @@ class SyncManager:
|
|||
)
|
||||
self._update_status(
|
||||
phase="estimates",
|
||||
stage="Estimating new or changed pull requests",
|
||||
stage="Estimating merged changes",
|
||||
total=len(queue),
|
||||
)
|
||||
estimator: Final = Estimator(settings, complete, estimator_models)
|
||||
|
|
@ -521,7 +550,7 @@ class SyncManager:
|
|||
try:
|
||||
evidence: Final = await github.evidence(repo, pull)
|
||||
except SourceError as exc:
|
||||
unavailable: Final = await _unavailable_record(github, repo, pull, exc)
|
||||
unavailable: Final = await _unavailable_record(github, settings, repo, pull, exc)
|
||||
self._update_estimate_progress(unavailable["estimate"])
|
||||
return _ProcessedPull(index, unavailable, metadata_unavailable=True)
|
||||
estimate: Final = await _estimate_with_fallback(estimator, evidence)
|
||||
|
|
@ -531,7 +560,13 @@ class SyncManager:
|
|||
"number": evidence["number"],
|
||||
"title": evidence["title"],
|
||||
"body": evidence["body"],
|
||||
"head": MappingProxyType({"sha": evidence["head_sha"]}),
|
||||
"head": MappingProxyType(
|
||||
{
|
||||
"sha": evidence["head_sha"],
|
||||
"ref": evidence.get("source_branch", ""),
|
||||
"repo": pull.head.repo if pull.head is not None else None,
|
||||
}
|
||||
),
|
||||
"user": MappingProxyType({"login": evidence["login"]}),
|
||||
"merged_at": evidence["merged_at"],
|
||||
"updated_at": evidence["merged_at"],
|
||||
|
|
@ -559,7 +594,26 @@ class SyncManager:
|
|||
worker_task.cancel()
|
||||
await asyncio.gather(*workers, return_exceptions=True)
|
||||
processed_by_index: Final = _processed_records(processed)
|
||||
records: Final = tuple(processed_by_index[index] for index in range(len(queue)))
|
||||
branch_repos: Final = tuple(
|
||||
sorted(
|
||||
frozenset(
|
||||
(
|
||||
*(repository_tag(settings, repo) for repo in settings.repos),
|
||||
*(pull.get("source_repo", "") for pull in records),
|
||||
)
|
||||
)
|
||||
- {""}
|
||||
)
|
||||
)
|
||||
branch_spend: Final = await branch_spend_reader(start, end, branch_repos) if branch_spend_reader else None
|
||||
branch_fields: Final[_BranchSpendFields] = (
|
||||
{"branch_spend": branch_spend} if branch_spend is not None else {}
|
||||
)
|
||||
report: Final = ROIReport(
|
||||
source_provider=settings.source_provider,
|
||||
source_api_url=settings.source_api_url,
|
||||
**branch_fields,
|
||||
mode="live",
|
||||
start=start.isoformat(),
|
||||
end=end.isoformat(),
|
||||
|
|
@ -569,7 +623,7 @@ class SyncManager:
|
|||
estimator_prompt=settings.estimator_prompt,
|
||||
effort_basis="without_ai",
|
||||
spend=spend,
|
||||
pulls=tuple(processed_by_index[index] for index in range(len(queue))),
|
||||
pulls=records,
|
||||
settings_fingerprint=settings_fingerprint(settings),
|
||||
warnings=repositories.warnings,
|
||||
unavailable_repos=repositories.unavailable_repos,
|
||||
|
|
@ -605,6 +659,7 @@ class SyncManager:
|
|||
except SourceError as exc:
|
||||
self._update_status(phase="error", stage="Sync failed", error=str(exc))
|
||||
except Exception: # noqa: BLE001 - background job boundary records a safe failure for every source error
|
||||
verbose_proxy_logger.exception("ROI Calculator sync failed")
|
||||
self._update_status(
|
||||
phase="error",
|
||||
stage="Sync failed",
|
||||
|
|
@ -646,6 +701,8 @@ class SyncManager:
|
|||
def _cached_record(self, pull: ROIPullRecord) -> ROIPullRecord:
|
||||
estimate: Final = _ESTIMATE_ADAPTER.validate_python(MappingProxyType({**pull["estimate"], "cached": True}))
|
||||
return ROIPullRecord(
|
||||
source_repo=pull.get("source_repo", ""),
|
||||
source_branch=pull.get("source_branch", ""),
|
||||
repo=pull["repo"],
|
||||
number=pull["number"],
|
||||
title=pull["title"],
|
||||
|
|
@ -672,6 +729,8 @@ class SyncManager:
|
|||
key: str | None,
|
||||
) -> ROIPullRecord:
|
||||
return ROIPullRecord(
|
||||
source_repo=evidence.get("source_repo", ""),
|
||||
source_branch=evidence.get("source_branch", ""),
|
||||
repo=evidence["repo"],
|
||||
number=evidence["number"],
|
||||
title=evidence["title"],
|
||||
|
|
|
|||
|
|
@ -158,6 +158,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
|
|||
"autorouter_savings_estimate",
|
||||
"autorouter_baseline_observation",
|
||||
"used_client_oauth_token",
|
||||
"litellm_roi_estimator",
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -211,6 +212,7 @@ def _get_spend_logs_metadata(
|
|||
usage_object=None,
|
||||
guardrail_information=None,
|
||||
internal_call_origin=None,
|
||||
litellm_roi_estimator=False,
|
||||
eval_information=None,
|
||||
cold_storage_object_key=cold_storage_object_key,
|
||||
litellm_overhead_time_ms=None,
|
||||
|
|
@ -244,6 +246,7 @@ def _get_spend_logs_metadata(
|
|||
router_metadata=router_metadata,
|
||||
azure_spillover=azure_spillover,
|
||||
used_client_oauth_token=used_client_oauth_token,
|
||||
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
|
||||
)
|
||||
_raw_key: Final = clean_metadata.get("user_api_key")
|
||||
_trusted_hash: Final = metadata.get("user_api_key_hash")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from collections.abc import Mapping
|
|||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, StrictFloat, StrictInt, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, StrictFloat, StrictInt, ValidationInfo, field_validator
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
DEFAULT_PROMPT: Final = (
|
||||
|
|
@ -11,18 +11,22 @@ DEFAULT_PROMPT: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _normalize_login(value: str) -> str:
|
||||
def normalize_source_login(value: str, provider: str = "github") -> str:
|
||||
import re
|
||||
|
||||
login: Final = value.strip().casefold()
|
||||
if re.fullmatch(r"[A-Za-z0-9_\[\]-]+", login) is None:
|
||||
raise ValueError("Enter a valid GitHub username.")
|
||||
pattern: Final = r"[A-Za-z0-9_.-]+" if provider == "gitlab" else r"[A-Za-z0-9_\[\]-]+"
|
||||
if re.fullmatch(pattern, login) is None:
|
||||
raise ValueError("Enter a valid source-control username.")
|
||||
return login
|
||||
|
||||
|
||||
class ROISettings(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
source_provider: Literal["github", "gitlab"] = "github"
|
||||
gitlab_api_url: str = "https://gitlab.com/api/v4"
|
||||
gitlab_token: SecretStr = SecretStr("")
|
||||
github_api_url: str = "https://api.github.com"
|
||||
github_token: SecretStr = SecretStr("")
|
||||
estimator_key: SecretStr = SecretStr("")
|
||||
|
|
@ -33,6 +37,10 @@ class ROISettings(BaseModel):
|
|||
update_interval_minutes: float = Field(default=1440, ge=0, le=43200, allow_inf_nan=False)
|
||||
identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
@property
|
||||
def source_api_url(self) -> str:
|
||||
return self.gitlab_api_url if self.source_provider == "gitlab" else self.github_api_url
|
||||
|
||||
@field_validator("update_interval_minutes")
|
||||
@classmethod
|
||||
def validate_update_interval(cls, value: float) -> float:
|
||||
|
|
@ -40,14 +48,14 @@ class ROISettings(BaseModel):
|
|||
raise ValueError("Choose manual updates (0), or an interval of at least 5 minutes.")
|
||||
return value
|
||||
|
||||
@field_validator("github_api_url")
|
||||
@field_validator("github_api_url", "gitlab_api_url")
|
||||
@classmethod
|
||||
def normalize_github_api_url(cls, value: str) -> str:
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
normalized: Final[str] = value.strip().rstrip("/")
|
||||
if not normalized:
|
||||
raise ValueError("A GitHub API URL is required.")
|
||||
raise ValueError("A source API URL is required.")
|
||||
parsed: Final = urlsplit(normalized)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
|
|
@ -57,26 +65,30 @@ class ROISettings(BaseModel):
|
|||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ValueError("Use an HTTPS GitHub API URL without credentials, query, or fragment.")
|
||||
raise ValueError("Use an HTTPS source API URL without credentials, query, or fragment.")
|
||||
return normalized
|
||||
|
||||
@field_validator("repos")
|
||||
@classmethod
|
||||
def validate_repositories(cls, values: tuple[str, ...]) -> tuple[str, ...]:
|
||||
def validate_repositories(cls, values: tuple[str, ...], info: ValidationInfo) -> tuple[str, ...]:
|
||||
import re
|
||||
|
||||
normalized_values: Final = tuple(repo.strip().rstrip("/").removesuffix(".git") for repo in values)
|
||||
normalized: Final = tuple(
|
||||
repo for index, repo in enumerate(normalized_values) if repo not in normalized_values[:index]
|
||||
)
|
||||
pattern: Final = (
|
||||
r"[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+"
|
||||
if info.data.get("source_provider") == "gitlab"
|
||||
else r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+"
|
||||
)
|
||||
invalid_repositories: Final = tuple(
|
||||
repo
|
||||
for repo in normalized
|
||||
if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo) is None
|
||||
or any(part in (".", "..") for part in repo.split("/"))
|
||||
if re.fullmatch(pattern, repo) is None or any(part in (".", "..") for part in repo.split("/"))
|
||||
)
|
||||
if invalid_repositories:
|
||||
raise ValueError("Repositories must use owner/repo format.")
|
||||
raise ValueError("Use owner/repo format, or group/subgroup/project for GitLab.")
|
||||
return normalized
|
||||
|
||||
@field_validator("estimator_prompt")
|
||||
|
|
@ -89,24 +101,29 @@ class ROISettings(BaseModel):
|
|||
|
||||
@field_validator("identity_map")
|
||||
@classmethod
|
||||
def normalize_identity_map(cls, values: Mapping[str, str]) -> Mapping[str, str]:
|
||||
def normalize_identity_map(cls, values: Mapping[str, str], info: ValidationInfo) -> Mapping[str, str]:
|
||||
from litellm.proxy.roi_calculator.analytics import normalize_email
|
||||
|
||||
normalized: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
_normalize_login(login): normalize_email(address)
|
||||
normalize_source_login(
|
||||
login, "gitlab" if info.data.get("source_provider") == "gitlab" else "github"
|
||||
): normalize_email(address)
|
||||
for login, address in values.items()
|
||||
if normalize_email(address)
|
||||
}
|
||||
)
|
||||
if len(normalized) != len(values):
|
||||
raise ValueError("Each identity needs a GitHub username and a valid gateway email.")
|
||||
raise ValueError("Each identity needs a source-control username and a valid gateway email.")
|
||||
return normalized
|
||||
|
||||
|
||||
class ROISettingsUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_provider: Literal["github", "gitlab"] | None = None
|
||||
gitlab_api_url: str | None = None
|
||||
gitlab_token: str | None = None
|
||||
github_api_url: str | None = None
|
||||
github_token: str | None = None
|
||||
estimator_key: str | None = None
|
||||
|
|
@ -118,6 +135,9 @@ class ROISettingsUpdate(BaseModel):
|
|||
|
||||
|
||||
class ROISettingsResponse(BaseModel):
|
||||
source_provider: Literal["github", "gitlab"] = "github"
|
||||
gitlab_api_url: str = "https://gitlab.com/api/v4"
|
||||
has_gitlab_token: bool = False
|
||||
github_api_url: str
|
||||
repos: tuple[str, ...]
|
||||
estimator_model: str
|
||||
|
|
@ -180,6 +200,8 @@ class ROIEstimate(TypedDict):
|
|||
|
||||
|
||||
class ROIPullRecord(TypedDict):
|
||||
source_repo: NotRequired[ReadOnly[str]]
|
||||
source_branch: NotRequired[ReadOnly[str]]
|
||||
repo: ReadOnly[str]
|
||||
number: ReadOnly[int]
|
||||
title: ReadOnly[str]
|
||||
|
|
@ -199,7 +221,34 @@ class ROIPullRecord(TypedDict):
|
|||
cache_key: ReadOnly[str | None]
|
||||
|
||||
|
||||
class ROIBranchSpend(BaseModel):
|
||||
repo: str
|
||||
branch: str
|
||||
spend: float
|
||||
requests: int
|
||||
|
||||
|
||||
class ROIBranchAttribution(BaseModel):
|
||||
repo: str
|
||||
branch: str
|
||||
spend: float | None = None
|
||||
requests: int = 0
|
||||
status: Literal["matched", "unattributed", "ambiguous", "unavailable"] = "unattributed"
|
||||
|
||||
|
||||
class ROIBranchMetrics(BaseModel):
|
||||
spend: float = 0
|
||||
hours: float = 0
|
||||
cost_per_hour: float | None = None
|
||||
matched_pulls: int = 0
|
||||
total_tagged_spend: float = 0
|
||||
unlinked_spend: float = 0
|
||||
|
||||
|
||||
class ROIReport(TypedDict):
|
||||
source_api_url: NotRequired[ReadOnly[str]]
|
||||
source_provider: NotRequired[ReadOnly[Literal["github", "gitlab"]]]
|
||||
branch_spend: NotRequired[ReadOnly[tuple[ROIBranchSpend, ...]]]
|
||||
mode: ReadOnly[str]
|
||||
start: ReadOnly[str]
|
||||
end: ReadOnly[str]
|
||||
|
|
@ -232,6 +281,8 @@ class ROIPullCommit(TypedDict):
|
|||
|
||||
|
||||
class ROIPullEvidence(TypedDict):
|
||||
source_repo: NotRequired[ReadOnly[str]]
|
||||
source_branch: NotRequired[ReadOnly[str]]
|
||||
repo: ReadOnly[str]
|
||||
number: ReadOnly[int]
|
||||
title: ReadOnly[str]
|
||||
|
|
@ -273,6 +324,9 @@ class ROIPersonSummary(TypedDict):
|
|||
|
||||
|
||||
class ROIPullSummary(TypedDict):
|
||||
branch_cost: ReadOnly[ROIBranchAttribution]
|
||||
source_repo: NotRequired[ReadOnly[str]]
|
||||
source_branch: NotRequired[ReadOnly[str]]
|
||||
repo: ReadOnly[str]
|
||||
number: ReadOnly[int]
|
||||
title: ReadOnly[str]
|
||||
|
|
@ -318,6 +372,9 @@ class ROITrendDay(TypedDict):
|
|||
|
||||
|
||||
class ROISummary(TypedDict):
|
||||
source_provider: ReadOnly[Literal["github", "gitlab"]]
|
||||
branch_metrics: ReadOnly[ROIBranchMetrics]
|
||||
unlinked_branches: ReadOnly[tuple[ROIBranchSpend, ...]]
|
||||
id: ReadOnly[str | None]
|
||||
mode: ReadOnly[str]
|
||||
start: ReadOnly[str]
|
||||
|
|
@ -375,6 +432,9 @@ class ROIEstimateResponse(BaseModel):
|
|||
|
||||
|
||||
class ROIPullResponse(BaseModel):
|
||||
source_repo: str = ""
|
||||
source_branch: str = ""
|
||||
branch_cost: ROIBranchAttribution = Field(default_factory=lambda: ROIBranchAttribution(repo="", branch=""))
|
||||
repo: str
|
||||
number: int
|
||||
title: str
|
||||
|
|
@ -404,6 +464,9 @@ class ROITrendResponse(BaseModel):
|
|||
|
||||
|
||||
class ROISummaryResponse(BaseModel):
|
||||
source_provider: Literal["github", "gitlab"] = "github"
|
||||
branch_metrics: ROIBranchMetrics = Field(default_factory=ROIBranchMetrics)
|
||||
unlinked_branches: tuple[ROIBranchSpend, ...] = ()
|
||||
id: str | None
|
||||
mode: str
|
||||
start: str
|
||||
|
|
@ -431,7 +494,7 @@ class ROIIdentityMapUpdate(BaseModel):
|
|||
@field_validator("github_login")
|
||||
@classmethod
|
||||
def normalize_login(cls, value: str) -> str:
|
||||
return _normalize_login(value)
|
||||
return value.strip().casefold()
|
||||
|
||||
|
||||
class ROIIdentityMapResponse(BaseModel):
|
||||
|
|
@ -478,7 +541,6 @@ class ROICompletionMessage(TypedDict):
|
|||
|
||||
class ROICompletionMetadata(TypedDict):
|
||||
tags: ReadOnly[tuple[str, ...]]
|
||||
litellm_roi_estimator: ReadOnly[bool]
|
||||
|
||||
|
||||
class ROIResponseFormat(TypedDict):
|
||||
|
|
|
|||
79
tests/integration/spend/test_roi_branch_spend.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import json
|
||||
import os
|
||||
import uuid
|
||||
from datetime import date
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from prisma import Prisma
|
||||
from psycopg import sql
|
||||
|
||||
from litellm.proxy.roi_calculator.branch_spend import read_branch_spend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() -> None:
|
||||
schema: Final = f"integration_roi_{uuid.uuid4().hex}"
|
||||
url: Final = os.environ["DATABASE_URL"]
|
||||
parsed: Final = urlsplit(url)
|
||||
scoped: Final = urlunsplit(parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "schema": schema})))
|
||||
repo: Final = "gitlab.com/group/project"
|
||||
tags: Final = (f"repo:{repo}", "branch:feature/one")
|
||||
rows: Final = (
|
||||
("2026-09-01 00:00:00", 2, tags),
|
||||
("2026-09-30 23:59:59.999", 3, tags + tags),
|
||||
("2026-10-01 00:00:00", 100, tags),
|
||||
("2026-08-31 23:59:59.999", 100, tags),
|
||||
("2026-09-15 00:00:00", 100, tags + ("branch:conflict",)),
|
||||
("2026-09-15 00:00:00", 100, tags + ("repo:gitlab.com/other/project",)),
|
||||
("2026-09-15 00:00:00", 100, ("branch:feature/one",)),
|
||||
("2026-09-15 00:00:00", 11, tags + ("litellm-roi-estimator",)),
|
||||
("2026-09-15 00:00:00", 0, (f"repo:{repo}", "branch:free")),
|
||||
("2026-09-15 00:00:00", 7, (f"repo:{repo}", "branch:Feature/one")),
|
||||
)
|
||||
with psycopg.connect(url, autocommit=True) as setup:
|
||||
setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
|
||||
try:
|
||||
setup.execute(
|
||||
sql.SQL(
|
||||
'CREATE TABLE {}."LiteLLM_SpendLogs" '
|
||||
'("startTime" timestamp, spend float, request_tags jsonb, metadata jsonb)'
|
||||
).format(sql.Identifier(schema))
|
||||
)
|
||||
for timestamp, spend, request_tags in rows:
|
||||
setup.execute(
|
||||
sql.SQL(
|
||||
'INSERT INTO {}."LiteLLM_SpendLogs" ("startTime", spend, request_tags) '
|
||||
'VALUES (%s::timestamp, %s, %s::jsonb)'
|
||||
).format(sql.Identifier(schema)),
|
||||
(timestamp, spend, json.dumps(request_tags)),
|
||||
)
|
||||
for marker, spend, extra_tags in (
|
||||
(True, 100, ()),
|
||||
(True, 100, ("litellm-roi-estimator",)),
|
||||
(False, 13, ("litellm-roi-estimator",)),
|
||||
(None, 100, ("litellm-roi-estimator",)),
|
||||
):
|
||||
setup.execute(
|
||||
sql.SQL('INSERT INTO {}."LiteLLM_SpendLogs" VALUES (%s::timestamp, %s, %s::jsonb, %s::jsonb)').format(
|
||||
sql.Identifier(schema)
|
||||
),
|
||||
(
|
||||
"2026-09-15 00:00:00",
|
||||
spend,
|
||||
json.dumps(tags + extra_tags),
|
||||
json.dumps({"litellm_roi_estimator": marker}),
|
||||
),
|
||||
)
|
||||
database: Final = Prisma(datasource={"url": scoped})
|
||||
await database.connect()
|
||||
try:
|
||||
result: Final = await read_branch_spend(database, date(2026, 9, 1), date(2026, 9, 30), (repo,))
|
||||
finally:
|
||||
await database.disconnect()
|
||||
costs: Final = {row.branch: (row.spend, row.requests) for row in result}
|
||||
assert costs == {"feature/one": (18, 3), "Feature/one": (7, 1), "free": (0, 1)}
|
||||
finally:
|
||||
setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
|
||||
|
|
@ -2,20 +2,25 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from math import isclose
|
||||
from types import MappingProxyType
|
||||
from typing import Final, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
from litellm.proxy.management_endpoints.roi_calculator_endpoints import (
|
||||
_estimator_models_from_deployments,
|
||||
_gateway_transport,
|
||||
_next_update,
|
||||
get_github_transport,
|
||||
get_roi_config_repository,
|
||||
register_scheduled_sync,
|
||||
router,
|
||||
|
|
@ -23,11 +28,58 @@ from litellm.proxy.management_endpoints.roi_calculator_endpoints import (
|
|||
)
|
||||
from litellm.proxy.roi_calculator.estimator import estimator_options
|
||||
from litellm.proxy.roi_calculator.sample import sample_report
|
||||
from litellm.types.roi_calculator import ROIReport, ROISettings, ROISyncStatus
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.types.roi_calculator import ROIReport, ROISettings, ROISummaryResponse, ROISyncStatus
|
||||
|
||||
_JSON_HEADERS: Final = MappingProxyType({"content-type": "application/json"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/v1/chat/completions", "/v1/responses", "/v1/messages"))
|
||||
@pytest.mark.parametrize("string_metadata", (False, True))
|
||||
async def test_only_internal_estimator_transport_can_mark_persisted_spend(path: str, string_metadata: bool) -> None:
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
app: Final = FastAPI()
|
||||
tags: Final = ("repo:org/repo", "branch:feature", "litellm-roi-estimator")
|
||||
forged: Final = {"tags": tags, "litellm_roi_estimator": True}
|
||||
metadata: Final = json.dumps(forged) if string_metadata else forged
|
||||
body: Final = {"model": "test-model", "metadata": metadata, "litellm_metadata": metadata}
|
||||
now: Final = datetime(2026, 9, 15, tzinfo=timezone.utc)
|
||||
|
||||
@app.post(path)
|
||||
async def log_request(request: Request) -> Mapping[str, object]:
|
||||
data: Final = await add_litellm_data_to_request(
|
||||
data=await request.json(),
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", metadata={"litellm_roi_estimator": True}),
|
||||
proxy_config=ProxyConfig(),
|
||||
)
|
||||
payload: Final = get_logging_payload(
|
||||
kwargs={"model": "test-model", "response_cost": 0.25, "litellm_params": data},
|
||||
response_obj={"id": "test-request", "usage": {"prompt_tokens": 10, "completion_tokens": 5}},
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
return {
|
||||
"metadata": json.loads(payload["metadata"]),
|
||||
"tags": json.loads(payload["request_tags"]),
|
||||
"spend": payload["spend"],
|
||||
}
|
||||
|
||||
async with (
|
||||
httpx.AsyncClient(transport=_gateway_transport(app), base_url="http://test") as internal,
|
||||
httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as external,
|
||||
):
|
||||
for client, expected in ((external, False), (internal, True), (external, False)):
|
||||
response: Final = await client.post(path, json=body, headers={"x-litellm-roi-estimator": "true"})
|
||||
assert response.status_code == 200
|
||||
logged: Final = response.json()
|
||||
assert logged["metadata"].get("litellm_roi_estimator") is expected
|
||||
assert set(logged["tags"]) == set(tags)
|
||||
assert logged["spend"] == 0.25
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_startup_keeps_one_roi_schedule() -> None:
|
||||
scheduler: Final = AsyncIOScheduler()
|
||||
|
|
@ -60,7 +112,7 @@ class _ConfigRepository:
|
|||
|
||||
async def get_param(self, param_name: str) -> _Parameter | None:
|
||||
value: Final = self.values.get(param_name)
|
||||
return _Parameter(value) if value is not None else None
|
||||
return _Parameter(value) if param_name in self.values else None
|
||||
|
||||
async def set_param(self, param_name: str, param_value: object) -> object:
|
||||
_assert_json_round_trip(param_value)
|
||||
|
|
@ -68,11 +120,14 @@ class _ConfigRepository:
|
|||
return self.values[param_name]
|
||||
|
||||
|
||||
def _client(role: LitellmUserRoles, repository: _ConfigRepository) -> TestClient:
|
||||
def _client(
|
||||
role: LitellmUserRoles, repository: _ConfigRepository, transport: httpx.AsyncBaseTransport | None = None
|
||||
) -> TestClient:
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role)
|
||||
app.dependency_overrides[get_roi_config_repository] = lambda: repository
|
||||
app.dependency_overrides[get_github_transport] = lambda: transport
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
|
|
@ -161,6 +216,42 @@ def test_github_api_url_must_use_https() -> None:
|
|||
assert not repository.values
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patch", ({"github_api_url": None}, {"gitlab_api_url": None}, {"repos": ["invalid"]}, {"estimator_prompt": " "})
|
||||
)
|
||||
def test_invalid_connection_settings_are_rejected_without_saving(patch: Mapping[str, object]) -> None:
|
||||
repository: Final = _ConfigRepository()
|
||||
client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository)
|
||||
assert client.put("/roi-calculator/settings", json=patch).status_code == 422
|
||||
assert not repository.values
|
||||
|
||||
|
||||
@pytest.mark.parametrize("upstream_status", (200, 403))
|
||||
def test_public_gitlab_repository_browser_and_errors(upstream_status: int) -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/api/v4/projects"
|
||||
assert request.url.params["search"] == "gateway"
|
||||
assert "PRIVATE-TOKEN" not in request.headers
|
||||
return httpx.Response(
|
||||
upstream_status, json=[{"id": 1, "path_with_namespace": "group/gateway"}], headers={"x-next-page": "2"}
|
||||
)
|
||||
|
||||
repository: Final = _ConfigRepository()
|
||||
client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository, httpx.MockTransport(respond))
|
||||
assert client.put("/roi-calculator/settings", json={"source_provider": "gitlab"}).status_code == 200
|
||||
response: Final = client.get("/roi-calculator/repositories", params={"query": "gateway"})
|
||||
if upstream_status == 200:
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"repositories": [{"name": "group/gateway", "visibility": "private", "archived": False}],
|
||||
"page": 1,
|
||||
"has_more": True,
|
||||
}
|
||||
else:
|
||||
assert response.status_code == 502
|
||||
assert "HTTP 403" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY])
|
||||
@pytest.mark.parametrize(
|
||||
"method,path,body",
|
||||
|
|
@ -209,8 +300,16 @@ def test_sample_preview_does_not_change_live_settings_or_report() -> None:
|
|||
client: Final = _client(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, repository)
|
||||
response: Final = client.get("/roi-calculator/report", params={"mode": "demo"})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["report"]["mode"] == "demo"
|
||||
assert response.json()["report"]["metrics"]["cost_per_hour"] > 0
|
||||
report: Final = ROISummaryResponse.model_validate(response.json()["report"])
|
||||
assert report.mode == "demo"
|
||||
assert report.metrics.cost_per_hour is not None and report.metrics.cost_per_hour > 0
|
||||
assert all(pull.branch_cost.status == "matched" and (pull.branch_cost.spend or 0) > 0 for pull in report.pulls)
|
||||
assert any(not pull.matched for pull in report.pulls)
|
||||
assert isclose(report.branch_metrics.spend, sum(pull.branch_cost.spend or 0 for pull in report.pulls))
|
||||
assert report.branch_metrics.unlinked_spend > 0
|
||||
assert isclose(
|
||||
report.branch_metrics.total_tagged_spend, report.branch_metrics.spend + report.branch_metrics.unlinked_spend
|
||||
)
|
||||
assert not repository.values
|
||||
assert client.get("/roi-calculator/report").json()["report"] is None
|
||||
|
||||
|
|
@ -264,3 +363,51 @@ def test_manual_match_recalculates_saved_report_and_removal_restores_cohort() ->
|
|||
assert removed.status_code == 200
|
||||
assert not removed.json()["identity_map"]
|
||||
assert removed.json()["report"]["metrics"] == before.json()["report"]["metrics"]
|
||||
|
||||
|
||||
def test_switching_sources_clears_report_and_identities_and_keeps_tokens_private(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "roi-calculator-test-salt-key-0123456789")
|
||||
repository: Final = _ConfigRepository()
|
||||
client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository)
|
||||
saved: Final = client.put(
|
||||
"/roi-calculator/settings",
|
||||
json={"source_provider": "gitlab", "gitlab_token": "private-gitlab-test", "repos": ["group/subgroup/project"]},
|
||||
)
|
||||
assert saved.status_code == 200
|
||||
assert saved.json()["has_gitlab_token"] is True
|
||||
assert "private-gitlab-test" not in saved.text
|
||||
assert "private-gitlab-test" not in str(repository.values)
|
||||
assert client.get("/roi-calculator/report").json()["report"] is None
|
||||
matched: Final = client.put(
|
||||
"/roi-calculator/identity-map", json={"github_login": "dev.name", "email": "dev@example.test"}
|
||||
)
|
||||
assert matched.status_code == 200
|
||||
assert matched.json()["identity_map"] == {"dev.name": "dev@example.test"}
|
||||
switched: Final = client.put("/roi-calculator/settings", json={"source_provider": "github"})
|
||||
assert switched.status_code == 200
|
||||
assert switched.json()["identity_map"] == {}
|
||||
assert switched.json()["repos"] == []
|
||||
assert client.get("/roi-calculator/report").json()["report"] is None
|
||||
changed_host: Final = client.put(
|
||||
"/roi-calculator/settings",
|
||||
json={"source_provider": "gitlab", "gitlab_api_url": "https://git.example.test/api/v4"},
|
||||
)
|
||||
assert changed_host.json()["has_gitlab_token"] is False
|
||||
|
||||
|
||||
def test_old_source_report_is_not_returned_when_matching_new_source_identity() -> None:
|
||||
repository: Final = _ConfigRepository()
|
||||
client: Final = _client(LitellmUserRoles.PROXY_ADMIN, repository)
|
||||
assert client.put("/roi-calculator/settings", json={"source_provider": "gitlab"}).status_code == 200
|
||||
old_report: Final = sample_report(datetime.now(timezone.utc))
|
||||
serialized: Final = TypeAdapter(dict[str, object]).validate_json(TypeAdapter(ROIReport).dump_json(old_report))
|
||||
asyncio.run(repository.set_param("roi_calculator_report", serialized))
|
||||
assert client.get("/roi-calculator/report").json()["report"] is None
|
||||
matched: Final = client.put(
|
||||
"/roi-calculator/identity-map", json={"github_login": "dev.name", "email": "dev@example.test"}
|
||||
)
|
||||
assert matched.status_code == 200
|
||||
assert matched.json()["report"] is None
|
||||
assert matched.json()["identity_map"] == {"dev.name": "dev@example.test"}
|
||||
|
|
|
|||
|
|
@ -145,3 +145,45 @@ def test_email_normalization_rejects_private_or_unusable_addresses() -> None:
|
|||
assert normalize_email("123+alice@users.noreply.github.com") == ""
|
||||
assert normalize_email("alice") == ""
|
||||
assert normalize_email("") == ""
|
||||
|
||||
|
||||
def test_branch_costs_are_independent_of_identity_and_never_count_reused_branches_twice() -> None:
|
||||
from litellm.types.roi_calculator import ROIBranchSpend
|
||||
|
||||
base: Final = _pull(emails=())
|
||||
pulls: Final[tuple[ROIPullRecord, ...]] = (
|
||||
{**base, "number": 1, "source_repo": "gitlab.com/group/repo", "source_branch": "feature"},
|
||||
{**base, "number": 2, "source_repo": "gitlab.com/group/repo", "source_branch": "reused"},
|
||||
{**base, "number": 3, "source_repo": "gitlab.com/group/repo", "source_branch": "reused"},
|
||||
{**base, "number": 4, "source_repo": "gitlab.com/group/repo", "source_branch": "missing"},
|
||||
{**base, "number": 5, "source_repo": "gitlab.com/group/repo", "source_branch": "free"},
|
||||
{
|
||||
**_pull(emails=(), estimate_status="error", hours=None),
|
||||
"number": 6,
|
||||
"source_repo": "gitlab.com/group/repo",
|
||||
"source_branch": "pending",
|
||||
},
|
||||
)
|
||||
report: Final[ROIReport] = {
|
||||
**_report(pulls),
|
||||
"branch_spend": (
|
||||
ROIBranchSpend(repo="gitlab.com/group/repo", branch="feature", spend=12, requests=2),
|
||||
ROIBranchSpend(repo="gitlab.com/group/repo", branch="reused", spend=7, requests=1),
|
||||
ROIBranchSpend(repo="gitlab.com/group/repo", branch="free", spend=0, requests=1),
|
||||
ROIBranchSpend(repo="gitlab.com/group/repo", branch="pending", spend=9, requests=1),
|
||||
),
|
||||
}
|
||||
result: Final = summarize(report, EMPTY_IDENTITY_MAP)
|
||||
costs: Final = {pull["number"]: pull["branch_cost"] for pull in result["pulls"]}
|
||||
assert costs[1].spend == 12
|
||||
assert costs[2].status == costs[3].status == "ambiguous"
|
||||
assert costs[2].spend is None
|
||||
assert costs[4].spend is None and costs[4].status == "unattributed"
|
||||
assert costs[5].spend == 0 and costs[5].status == "matched"
|
||||
assert result["branch_metrics"].cost_per_hour == 12 / 8
|
||||
assert result["branch_metrics"].unlinked_spend == 16
|
||||
assert result["branch_metrics"].matched_pulls == 3
|
||||
assert result["branch_metrics"].spend == 12
|
||||
assert result["metrics"]["matched_spend"] == 0
|
||||
incomplete: Final = summarize({**report, "unavailable_repos": ("other/repo",)}, EMPTY_IDENTITY_MAP)
|
||||
assert incomplete["branch_metrics"].cost_per_hour is None
|
||||
|
|
|
|||
32
tests/unit/proxy/roi_calculator/test_branch_spend.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import json
|
||||
from datetime import date
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.roi_calculator.branch_spend import read_branch_spend
|
||||
from litellm.types.roi_calculator import ROIBranchSpend
|
||||
|
||||
|
||||
class _SpendDatabase:
|
||||
async def query_raw(self, query: str, *args: object) -> object:
|
||||
assert args == (
|
||||
"2026-01-31T00:00:00+00:00",
|
||||
"2026-02-01T00:00:00+00:00",
|
||||
json.dumps(("gitlab.com/group/project",)),
|
||||
False,
|
||||
)
|
||||
return [{"repo": "gitlab.com/group/project", "branch": "feature", "spend": 0.000027, "requests": 3}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_branch_spend_includes_the_final_utc_day_and_preserves_fractional_costs() -> None:
|
||||
result: Final = await read_branch_spend(
|
||||
_SpendDatabase(), date(2026, 1, 31), date(2026, 1, 31), ("gitlab.com/group/project",)
|
||||
)
|
||||
assert result == (ROIBranchSpend(repo="gitlab.com/group/project", branch="feature", spend=0.000027, requests=3),)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_repositories_returns_no_spend_without_querying_the_database() -> None:
|
||||
assert await read_branch_spend(_SpendDatabase(), date(2026, 1, 1), date(2026, 1, 31), ()) == ()
|
||||
281
tests/unit/proxy/roi_calculator/test_gitlab.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
import asyncio
|
||||
from datetime import date
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import SecretStr
|
||||
|
||||
from litellm.proxy.roi_calculator.estimator import metadata_evidence
|
||||
from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError
|
||||
from litellm.proxy.roi_calculator.gitlab import GitLab
|
||||
from litellm.types.roi_calculator import ROISettings
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_lookups_overlap_with_a_bounded_number_of_requests() -> None:
|
||||
started: Final[asyncio.Queue[int]] = asyncio.Queue()
|
||||
release: Final = tuple(asyncio.Event() for _ in range(9))
|
||||
source_ids: Final = (*range(2, 11), 3)
|
||||
|
||||
async def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/projects/group/repo"):
|
||||
return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"})
|
||||
if request.url.path.endswith("/merge_requests"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=[
|
||||
{
|
||||
"iid": index,
|
||||
"title": "Fix parser",
|
||||
"web_url": f"https://gitlab.com/group/repo/-/merge_requests/{index}",
|
||||
"author": {"username": "dev"},
|
||||
"merged_at": "2026-09-30T12:00:00Z",
|
||||
"updated_at": "2026-09-30T12:00:00Z",
|
||||
"source_branch": f"fix/{index}",
|
||||
"source_project_id": source_id,
|
||||
}
|
||||
for index, source_id in enumerate(source_ids)
|
||||
],
|
||||
)
|
||||
project_id: Final = int(request.url.path.rsplit("/", 1)[1])
|
||||
started.put_nowait(project_id)
|
||||
await release[project_id - 2].wait()
|
||||
if project_id == 3:
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(200, json={"id": project_id, "path_with_namespace": f"fork-{project_id}/repo"})
|
||||
|
||||
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
|
||||
pending: Final = asyncio.create_task(source.pulls("group/repo", date(2026, 9, 1), date(2026, 9, 30)))
|
||||
try:
|
||||
first_wave: Final = tuple([await asyncio.wait_for(started.get(), timeout=1) for _ in range(8)])
|
||||
assert len(set(first_wave)) == 8
|
||||
assert started.empty()
|
||||
release[first_wave[0] - 2].set()
|
||||
next_id: Final = await asyncio.wait_for(started.get(), timeout=1)
|
||||
assert next_id not in first_wave
|
||||
for event in release:
|
||||
event.set()
|
||||
pulls: Final = await asyncio.wait_for(pending, timeout=1)
|
||||
assert tuple(pull.head.repo.full_name if pull.head and pull.head.repo else None for pull in pulls) == tuple(
|
||||
None if source_id == 3 else f"fork-{source_id}/repo" for source_id in source_ids
|
||||
)
|
||||
assert started.empty()
|
||||
finally:
|
||||
for event in release:
|
||||
event.set()
|
||||
pending.cancel()
|
||||
await asyncio.gather(pending, return_exceptions=True)
|
||||
await source.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("missing_fork,source_id", ((False, 2), (True, 2), (False, None)))
|
||||
async def test_gitlab_paginates_nested_projects_and_keeps_source_code_out_of_estimates(
|
||||
missing_fork: bool, source_id: int | None
|
||||
) -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.headers["PRIVATE-TOKEN"] == "test-only-token"
|
||||
assert request.url.host == "git.example.test"
|
||||
path: Final = request.url.path
|
||||
detail: Final = {
|
||||
"iid": 8,
|
||||
"title": "Fix parser",
|
||||
"description": "Handle empty input",
|
||||
"web_url": "https://git.example.test/g/sub/p/-/merge_requests/8",
|
||||
"author": {"username": "dev.name"},
|
||||
"merged_at": "2026-09-30T23:59:59Z",
|
||||
"updated_at": "2026-10-01T00:00:00Z",
|
||||
"sha": "sha",
|
||||
"source_branch": "fix/parser",
|
||||
"source_project_id": source_id,
|
||||
"changes_count": "1",
|
||||
}
|
||||
if path.endswith("/projects/g/sub/p"):
|
||||
assert "%2F" in str(request.url)
|
||||
return httpx.Response(200, json={"id": 1, "path_with_namespace": "g/sub/p"})
|
||||
if path.endswith("/projects/2"):
|
||||
return (
|
||||
httpx.Response(404)
|
||||
if missing_fork
|
||||
else httpx.Response(200, json={"id": 2, "path_with_namespace": "dev/fork"})
|
||||
)
|
||||
if path.endswith("/merge_requests"):
|
||||
assert request.url.params["scope"] == "all"
|
||||
if request.url.params["page"] == "1":
|
||||
return httpx.Response(
|
||||
200, json=[{**detail, "iid": 7, "merged_at": "2026-10-01T00:00:00Z"}], headers={"x-next-page": "2"}
|
||||
)
|
||||
return httpx.Response(200, json=[detail])
|
||||
if path.endswith("/merge_requests/8"):
|
||||
return httpx.Response(200, json=detail)
|
||||
if path.endswith("/diffs"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=[
|
||||
{
|
||||
"new_path": "parser.py",
|
||||
"old_path": "parser.py",
|
||||
"diff": "@@ -1 +1 @@\n---old-code\n+++private-code",
|
||||
}
|
||||
],
|
||||
)
|
||||
if path.endswith("/commits"):
|
||||
return httpx.Response(
|
||||
200, json=[{"id": "sha", "message": "Fix empty input", "author_email": "untrusted@example.test"}]
|
||||
)
|
||||
if path.endswith("/users"):
|
||||
return httpx.Response(200, json=[{"username": "dev.name", "public_email": "dev@example.test"}])
|
||||
raise AssertionError(path)
|
||||
|
||||
settings: Final = ROISettings(
|
||||
source_provider="gitlab",
|
||||
gitlab_api_url="https://git.example.test/api/v4",
|
||||
gitlab_token=SecretStr("test-only-token"),
|
||||
repos=("g/sub/p",),
|
||||
)
|
||||
client: Final = GitLab(settings, httpx.MockTransport(respond))
|
||||
try:
|
||||
pulls: Final = await client.pulls("g/sub/p", date(2026, 9, 1), date(2026, 9, 30))
|
||||
assert tuple(pull.number for pull in pulls) == (8,)
|
||||
evidence: Final = await client.evidence("g/sub/p", pulls[0])
|
||||
assert evidence["source_repo"] == ("" if missing_fork or source_id is None else "git.example.test/dev/fork")
|
||||
assert evidence["source_branch"] == "fix/parser"
|
||||
assert evidence["emails"] == ("dev@example.test",)
|
||||
assert evidence["commit_emails"] == ()
|
||||
assert (evidence["additions"], evidence["deletions"]) == (1, 1)
|
||||
assert not evidence["incomplete_metadata"]
|
||||
assert "private-code" not in metadata_evidence(evidence).model_dump_json()
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", (301, 401, 403, 404))
|
||||
async def test_gitlab_errors_do_not_follow_redirects_or_disclose_upstream_content(status: int) -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.host == "gitlab.com"
|
||||
return httpx.Response(status, text="secret-upstream-response", headers={"location": "https://untrusted.test/"})
|
||||
|
||||
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
|
||||
try:
|
||||
with pytest.raises(SourceError, match=f"HTTP {status}") as error:
|
||||
await source.test_repositories(("group/project",))
|
||||
assert "secret-upstream-response" not in str(error.value)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("token", ("", "test-token"))
|
||||
async def test_gitlab_repository_browser_preserves_visibility_pagination_and_membership(token: str) -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.params["search"] == "gateway"
|
||||
assert request.url.params["page"] == "2"
|
||||
assert (request.url.params.get("membership") == "true") == bool(token)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=[
|
||||
{
|
||||
"id": 1,
|
||||
"path_with_namespace": "group/sub/gateway",
|
||||
"visibility": "internal",
|
||||
"archived": True,
|
||||
}
|
||||
],
|
||||
headers={"link": '<https://gitlab.com/api/v4/projects?page=3>; rel="next"'},
|
||||
)
|
||||
|
||||
source: Final = GitLab(
|
||||
ROISettings(source_provider="gitlab", gitlab_token=SecretStr(token)), httpx.MockTransport(respond)
|
||||
)
|
||||
try:
|
||||
assert await source.repositories("gateway", 2) == ((("group/sub/gateway", "internal", True),), True)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"resource,message",
|
||||
(
|
||||
("projects", "page of results"),
|
||||
("projects/group/repo", "project details"),
|
||||
("projects/1/merge_requests/8", "merge request details"),
|
||||
),
|
||||
)
|
||||
async def test_gitlab_rejects_malformed_responses(resource: str, message: str) -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/" + resource):
|
||||
return httpx.Response(200, json={"private-error": "must not be disclosed"})
|
||||
return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"})
|
||||
|
||||
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
|
||||
operation: Final = (
|
||||
source.repositories()
|
||||
if resource == "projects"
|
||||
else source.test_repositories(("group/repo",))
|
||||
if resource == "projects/group/repo"
|
||||
else source.evidence("group/repo", GitHubPullListItem(number=8, title="Fix", updated_at="2026-09-30"))
|
||||
)
|
||||
try:
|
||||
with pytest.raises(SourceError, match=message):
|
||||
await operation
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gitlab_connection_failure_is_sanitized_and_profile_uses_fallback() -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("private host detail", request=request)
|
||||
|
||||
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
|
||||
try:
|
||||
with pytest.raises(SourceError, match="Could not reach GitLab") as error:
|
||||
await source.repositories()
|
||||
assert "private host detail" not in str(error.value)
|
||||
assert await source.profile_email("alice", fallback="known@example.test") == "known@example.test"
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gitlab_stops_an_endless_pagination_response() -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/projects/group/repo"):
|
||||
return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"})
|
||||
assert int(request.url.params["page"]) <= 100
|
||||
return httpx.Response(200, json=[], headers={"x-next-page": "101"})
|
||||
|
||||
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
|
||||
try:
|
||||
with pytest.raises(SourceError, match="pagination limit"):
|
||||
await source.pulls("group/repo", date(2026, 9, 1), date(2026, 9, 30))
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gitlab_retries_transient_errors_and_checks_merge_request_access() -> None:
|
||||
statuses: Final = iter((429, 503, 200))
|
||||
reads: Final = iter(("/api/v4/projects/group/repo", "/api/v4/projects/1/merge_requests"))
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/projects/group/repo"):
|
||||
status: Final = next(statuses)
|
||||
if status != 200:
|
||||
return httpx.Response(status)
|
||||
assert request.url.path == next(reads)
|
||||
return httpx.Response(200, json={"id": 1, "path_with_namespace": "group/repo"})
|
||||
assert request.url.path == next(reads)
|
||||
assert request.url.params["state"] == "merged"
|
||||
return httpx.Response(200, json=[])
|
||||
|
||||
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
|
||||
try:
|
||||
await source.test_repositories(("group/repo",))
|
||||
assert next(reads, None) is None
|
||||
assert next(statuses, None) is None
|
||||
finally:
|
||||
await source.close()
|
||||
|
|
@ -14,6 +14,7 @@ from litellm.proxy.roi_calculator.estimator import CompletionCaller
|
|||
from litellm.proxy.roi_calculator.github import GitHubPullListItem
|
||||
from litellm.proxy.roi_calculator.sync import SpendReader, SyncManager, read_spend
|
||||
from litellm.types.roi_calculator import (
|
||||
ROIBranchSpend,
|
||||
ROICompletionRequest,
|
||||
ROIReport,
|
||||
ROISettings,
|
||||
|
|
@ -28,7 +29,7 @@ _PULL_LIST_JSON: Final = """[
|
|||
"body": "Preserve UTC behavior.",
|
||||
"merged_at": "2026-09-12T12:00:00Z",
|
||||
"updated_at": "2026-09-12T12:00:00Z",
|
||||
"head": {"sha": "abcdef"},
|
||||
"head": {"sha": "abcdef", "ref": "feature", "repo": {"full_name": "org/repo"}},
|
||||
"user": {"login": "alice"}
|
||||
}
|
||||
]"""
|
||||
|
|
@ -39,7 +40,7 @@ _PULL_DETAIL_JSON: Final = """{
|
|||
"html_url": "https://github.com/org/repo/pull/42",
|
||||
"user": {"login": "alice"},
|
||||
"merged_at": "2026-09-12T12:00:00Z",
|
||||
"head": {"sha": "abcdef"},
|
||||
"head": {"sha": "abcdef", "ref": "feature", "repo": {"full_name": "org/repo"}},
|
||||
"additions": 1,
|
||||
"deletions": 1,
|
||||
"changed_files": 1,
|
||||
|
|
@ -242,10 +243,116 @@ async def test_unchanged_estimated_pull_refreshes_identity_without_model_call()
|
|||
assert manager.status.reused == 1
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
assert report["pulls"][0]["estimate"].get("cached") is True
|
||||
assert report["pulls"][0]["source_branch"] == "feature"
|
||||
assert report["pulls"][0]["source_repo"] == "github.com/org/repo"
|
||||
assert report["pulls"][0]["profile_email"] == "new@example.com"
|
||||
assert report["pulls"][0]["emails"] == ("alice@example.com", "new@example.com")
|
||||
|
||||
|
||||
def _gitlab_transport(source_path: str | None, *, details_fail: bool = False) -> httpx.MockTransport:
|
||||
detail: Final = {
|
||||
"iid": 42,
|
||||
"title": "Fix timezone conversion",
|
||||
"description": "Preserve UTC behavior.",
|
||||
"web_url": "https://gitlab.com/org/repo/-/merge_requests/42",
|
||||
"author": {"username": "alice"},
|
||||
"merged_at": "2026-09-12T12:00:00Z",
|
||||
"updated_at": "2026-09-12T12:00:00Z",
|
||||
"sha": "abcdef",
|
||||
"source_branch": "feature",
|
||||
"source_project_id": 2,
|
||||
"changes_count": "1",
|
||||
}
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
path: Final = request.url.path
|
||||
if path.endswith("/projects/org/repo"):
|
||||
return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/repo"})
|
||||
if path.endswith("/projects/2"):
|
||||
return (
|
||||
httpx.Response(200, json={"id": 2, "path_with_namespace": source_path})
|
||||
if source_path
|
||||
else httpx.Response(404)
|
||||
)
|
||||
if path.endswith("/merge_requests"):
|
||||
return httpx.Response(
|
||||
200, json=[detail, {**detail, "iid": 43, "source_branch": "other"}] if details_fail else [detail]
|
||||
)
|
||||
if path.endswith("/merge_requests/43"):
|
||||
return httpx.Response(200, json={**detail, "iid": 43, "source_branch": "other"})
|
||||
if path.endswith("/merge_requests/42"):
|
||||
return httpx.Response(404) if details_fail else httpx.Response(200, json=detail)
|
||||
if path.endswith("/diffs"):
|
||||
return httpx.Response(200, json=[{"new_path": "time.py", "old_path": "time.py", "diff": "+fixed"}])
|
||||
if path.endswith("/commits"):
|
||||
return httpx.Response(200, json=[{"id": "abcdef", "message": "Fix timezone conversion"}])
|
||||
if path.endswith("/users"):
|
||||
return httpx.Response(200, json=[{"username": "alice", "public_email": "alice@example.com"}])
|
||||
raise AssertionError(path)
|
||||
|
||||
return httpx.MockTransport(respond)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("before,after", [(None, "dev/fork"), ("dev/fork", None), ("dev/fork", "dev/renamed")])
|
||||
async def test_gitlab_cache_refreshes_branch_attribution_when_source_access_changes(
|
||||
before: str | None, after: str | None
|
||||
) -> None:
|
||||
settings: Final = _settings().model_copy(update={"source_provider": "gitlab"})
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
|
||||
async def branch_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]:
|
||||
return (ROIBranchSpend(repo="gitlab.com/" + (after or "dev/fork"), branch="feature", spend=2.5, requests=3),)
|
||||
|
||||
assert await manager.start(settings, repository, _spend_reader(), _completion(), _gitlab_transport(before))
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
assert await manager.start(
|
||||
settings, repository, _spend_reader(), _completion(), _gitlab_transport(after), branch_spend_reader=branch_spend
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
assert report["pulls"][0]["source_repo"] == ("gitlab.com/" + after if after else "")
|
||||
result: Final = summarize(report, {})
|
||||
assert result["pulls"][0]["branch_cost"].status == ("matched" if after else "unattributed")
|
||||
|
||||
async def unexpected_completion(request: ROICompletionRequest) -> object:
|
||||
raise AssertionError("Unchanged source metadata must reuse the estimate")
|
||||
|
||||
assert await manager.start(settings, repository, _spend_reader(), unexpected_completion, _gitlab_transport(after))
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
assert manager.status.reused == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unreadable_gitlab_details_keep_known_branch_costs() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
settings: Final = _settings().model_copy(update={"source_provider": "gitlab"})
|
||||
|
||||
async def branch_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]:
|
||||
return (ROIBranchSpend(repo="gitlab.com/dev/fork", branch="feature", spend=2.5, requests=3),)
|
||||
|
||||
assert await manager.start(
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_gitlab_transport("dev/fork", details_fail=True),
|
||||
branch_spend_reader=branch_spend,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
result: Final = summarize(report, {})
|
||||
assert result["pulls"][0]["branch_cost"].spend == 2.5
|
||||
assert result["pulls"][0]["estimate"]["status"] == "needs_review"
|
||||
assert result["branch_metrics"].matched_pulls == 1
|
||||
assert result["branch_metrics"].cost_per_hour is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_spend_joins_user_emails_and_preserves_unmatched_identities() -> None:
|
||||
spend: Final = await read_spend(
|
||||
|
|
@ -461,6 +568,8 @@ async def test_one_unreadable_pr_preserves_other_estimates_in_report() -> None:
|
|||
assert manager.status.phase == "complete"
|
||||
assert manager.status.estimated == 1
|
||||
assert manager.status.needs_attention == 1
|
||||
assert report["pulls"][1]["source_repo"] == "github.com/org/repo"
|
||||
assert report["pulls"][1]["source_branch"] == "feature"
|
||||
|
||||
|
||||
def _repository_outage_transport(
|
||||
|
|
|
|||
|
|
@ -3530,6 +3530,24 @@ def test_get_spend_logs_metadata_keeps_user_agent():
|
|||
assert _get_spend_logs_metadata(None)["user_agent"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metadata,expected",
|
||||
(
|
||||
(None, False),
|
||||
({}, False),
|
||||
({"tags": ["litellm-roi-estimator"]}, False),
|
||||
({"litellm_roi_estimator": None}, False),
|
||||
({"litellm_roi_estimator": "true"}, False),
|
||||
({"litellm_roi_estimator": False}, False),
|
||||
({"litellm_roi_estimator": True}, True),
|
||||
),
|
||||
)
|
||||
def test_new_spend_logs_always_have_an_explicit_roi_estimator_marker(
|
||||
metadata: dict[str, object] | None, expected: bool
|
||||
) -> None:
|
||||
assert _get_spend_logs_metadata(metadata)["litellm_roi_estimator"] is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"client_sent_oauth_token, custom_llm_provider, expected",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import {
|
|||
} from "@/components/ui/dialog";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { effortNote, estimateLabel } from "./roiCalculatorData";
|
||||
import { effortNote, estimateLabel, branchCostLabel } from "./roiCalculatorData";
|
||||
import type { ROIIdentityMapUpdate, ROIPull, ROISummary } from "./roiCalculatorData";
|
||||
import type { ROIPerson } from "./roiCalculatorData";
|
||||
|
||||
|
|
@ -29,28 +29,37 @@ export function PullReasoningDialog({
|
|||
summary: ROISummary | null;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const titleRef = React.useRef<HTMLHeadingElement>(null);
|
||||
return (
|
||||
<Dialog open={Boolean(pull)} onOpenChange={(open) => !open && onClose()}>
|
||||
<DialogContent className="max-h-[85dvh] overflow-y-auto sm:max-w-xl">
|
||||
<DialogContent className="max-h-[85dvh] overflow-y-auto sm:max-w-2xl" initialFocus={titleRef}>
|
||||
{pull && (
|
||||
<>
|
||||
<DialogHeader>
|
||||
<DialogTitle>{pull.title}</DialogTitle>
|
||||
<DialogTitle ref={titleRef} tabIndex={-1} className="pr-8 text-lg leading-6">
|
||||
{pull.title}
|
||||
</DialogTitle>
|
||||
<DialogDescription>
|
||||
{pull.repo} #{pull.number} · {pull.login}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div>
|
||||
<p className="text-sm text-muted-foreground">Estimated engineering hours</p>
|
||||
<p className="mt-2 text-3xl font-semibold tabular-nums">{estimateLabel(pull.estimate)}</p>
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
<div className="space-y-3">
|
||||
<dl className="grid grid-cols-2 gap-4 rounded-lg border bg-muted/20 p-4">
|
||||
<div className="space-y-2">
|
||||
<dt className="text-xs text-muted-foreground">Estimated effort</dt>
|
||||
<dd className="text-2xl font-semibold tabular-nums">{estimateLabel(pull.estimate)}</dd>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<dt className="text-xs text-muted-foreground">Recorded AI cost</dt>
|
||||
<dd className="text-xl font-semibold tabular-nums">{branchCostLabel(pull)}</dd>
|
||||
{pull.branch_cost?.status === "matched" && (
|
||||
<dd className="text-xs text-muted-foreground">{pull.branch_cost.requests} requests</dd>
|
||||
)}
|
||||
</div>
|
||||
</dl>
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
{effortNote(pull.estimate.effort_basis ?? summary?.effort_basis)}
|
||||
</p>
|
||||
{pull.estimate.evidence_source === "pr_metadata" && (
|
||||
<p className="mt-1 text-xs text-muted-foreground">
|
||||
Based on PR descriptions, file change counts, and commit metadata.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<section>
|
||||
<h3 className="mb-2 font-medium">Reasoning</h3>
|
||||
|
|
@ -66,6 +75,34 @@ export function PullReasoningDialog({
|
|||
<dt className="text-muted-foreground">Email match</dt>
|
||||
<dd>{pull.email || "Not matched"}</dd>
|
||||
</dl>
|
||||
<section className="space-y-2 border-t pt-4 text-sm">
|
||||
<h3 className="font-medium">Track costs for this branch</h3>
|
||||
{pull.branch_cost?.status === "matched" && (
|
||||
<p className="text-muted-foreground">
|
||||
{pull.branch_cost.spend?.toFixed(8)} USD across {pull.branch_cost.requests} requests
|
||||
</p>
|
||||
)}
|
||||
{pull.source_repo && pull.source_branch ? (
|
||||
<>
|
||||
<p className="text-muted-foreground">Send both tags with each gateway request from this branch:</p>
|
||||
<pre className="overflow-x-auto rounded-md bg-muted p-3 text-xs">
|
||||
{JSON.stringify(
|
||||
{ metadata: { tags: [`repo:${pull.source_repo}`, `branch:${pull.source_branch}`] } },
|
||||
null,
|
||||
2,
|
||||
)}
|
||||
</pre>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Retained requests in the report’s UTC period. Branch names are case-sensitive. Reused branches
|
||||
cannot be split between changes.
|
||||
</p>
|
||||
</>
|
||||
) : (
|
||||
<p className="text-muted-foreground">
|
||||
The source repository or branch is unavailable. Sync again to refresh its metadata.
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
{summary?.estimator_prompt && (
|
||||
<details className="rounded-md border p-3">
|
||||
<summary className="cursor-pointer font-medium">Estimator prompt</summary>
|
||||
|
|
@ -80,7 +117,7 @@ export function PullReasoningDialog({
|
|||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
View on GitHub
|
||||
View on {summary?.source_provider === "gitlab" ? "GitLab" : "GitHub"}
|
||||
</a>
|
||||
)}
|
||||
</DialogFooter>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
import { apiClient } from "@/components/networking";
|
||||
import ROICalculatorView from "./ROICalculatorView";
|
||||
|
|
@ -13,21 +12,6 @@ vi.mock("@/components/networking", () => ({
|
|||
put: vi.fn(),
|
||||
},
|
||||
}));
|
||||
vi.mock("@/components/ui/chart", () => ({
|
||||
ChartContainer: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
ChartLegend: () => null,
|
||||
ChartLegendContent: () => null,
|
||||
ChartTooltip: () => null,
|
||||
ChartTooltipContent: () => null,
|
||||
}));
|
||||
vi.mock("recharts", () => ({
|
||||
Bar: () => null,
|
||||
CartesianGrid: () => null,
|
||||
ComposedChart: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
Line: () => null,
|
||||
XAxis: () => null,
|
||||
YAxis: () => null,
|
||||
}));
|
||||
|
||||
const summary = {
|
||||
id: null,
|
||||
|
|
@ -73,6 +57,15 @@ const summary = {
|
|||
pulls: [
|
||||
{
|
||||
repo: "org/repo",
|
||||
source_repo: "github.com/org/repo",
|
||||
source_branch: "feature/routing",
|
||||
branch_cost: {
|
||||
repo: "github.com/org/repo",
|
||||
branch: "feature/routing",
|
||||
status: "matched",
|
||||
spend: 8,
|
||||
requests: 12,
|
||||
},
|
||||
number: 42,
|
||||
title: "Improve request routing",
|
||||
url: "https://github.com/org/repo/pull/42",
|
||||
|
|
@ -131,6 +124,7 @@ const idleStatus = {
|
|||
|
||||
describe("ROICalculatorView", () => {
|
||||
beforeEach(() => {
|
||||
window.history.replaceState(null, "", "/roi-calculator/");
|
||||
vi.mocked(apiClient.get).mockReset();
|
||||
vi.mocked(apiClient.put).mockReset();
|
||||
vi.mocked(apiClient.post).mockReset();
|
||||
|
|
@ -145,8 +139,8 @@ describe("ROICalculatorView", () => {
|
|||
it("shows the spend summary and opens an accessible pull reasoning dialog", async () => {
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
|
||||
expect(await screen.findByText("Spend per estimated engineering hour")).toBeInTheDocument();
|
||||
expect(screen.getByText("$3.00")).toBeInTheDocument();
|
||||
expect(await screen.findByText("Gateway AI cost")).toBeInTheDocument();
|
||||
expect(screen.getByText("$20.00")).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Open estimate for org/repo pull request 42" }));
|
||||
|
||||
expect(await screen.findByRole("dialog")).toBeInTheDocument();
|
||||
|
|
@ -157,6 +151,19 @@ describe("ROICalculatorView", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("separates the overview, people, and branch reports into three tabs", async () => {
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByRole("heading", { name: "Where AI costs are matched" })).toBeVisible();
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Branches" }));
|
||||
expect(screen.getByRole("heading", { name: "Costs by branch" })).toBeVisible();
|
||||
expect(screen.getByRole("cell", { name: "$8.00" })).toBeVisible();
|
||||
fireEvent.click(screen.getByRole("tab", { name: "People" }));
|
||||
expect(screen.getByRole("heading", { name: "People and account matches" })).toBeVisible();
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Overview" }));
|
||||
expect(screen.getByRole("heading", { name: "Highest-cost changes" })).toBeVisible();
|
||||
expect(screen.queryByRole("radio")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows incomplete repository results without a spend-per-hour figure", async () => {
|
||||
const warning = "Incomplete report: could not read org/unavailable. Spend-per-hour figures are unavailable.";
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) => {
|
||||
|
|
@ -179,7 +186,8 @@ describe("ROICalculatorView", () => {
|
|||
expect(await screen.findByRole("alert")).toHaveTextContent(warning);
|
||||
expect(screen.getByRole("button", { name: "Open estimate for org/repo pull request 42" })).toBeInTheDocument();
|
||||
expect(screen.queryByText("$3.00")).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByText("Calculation details"));
|
||||
fireEvent.click(screen.getByRole("tab", { name: "People" }));
|
||||
fireEvent.click(screen.getByText("How this is calculated"));
|
||||
expect(
|
||||
screen.getByText("Spend per estimated hour is unavailable until all selected repositories can be read."),
|
||||
).toBeVisible();
|
||||
|
|
@ -201,7 +209,7 @@ describe("ROICalculatorView", () => {
|
|||
|
||||
render(<ROICalculatorView accessToken="token" userRole="Admin" isViewOnly />);
|
||||
|
||||
expect(await screen.findByText("Spend per estimated engineering hour")).toBeInTheDocument();
|
||||
expect(await screen.findByText("Gateway AI cost")).toBeInTheDocument();
|
||||
expect(screen.getByRole("note")).toHaveTextContent("Read-only access");
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Cancel sync" })).not.toBeInTheDocument();
|
||||
|
|
@ -210,8 +218,8 @@ describe("ROICalculatorView", () => {
|
|||
expect(screen.getByText("alice-work")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "alice-work" })).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Settings" }));
|
||||
expect(screen.getByLabelText("GitHub token")).toBeDisabled();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Settings" }));
|
||||
expect(screen.getByLabelText("GitHub token (optional for public repositories)")).toBeDisabled();
|
||||
expect(screen.queryByRole("button", { name: "Save settings" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
|
@ -244,9 +252,119 @@ describe("ROICalculatorView", () => {
|
|||
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
|
||||
expect(await screen.findByRole("heading", { name: "Connect GitHub to get started" })).toBeInTheDocument();
|
||||
expect(screen.getByLabelText("GitHub token")).toHaveAttribute("type", "password");
|
||||
expect(screen.getAllByText("Connect GitHub to get started")).toHaveLength(1);
|
||||
expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeInTheDocument();
|
||||
expect(screen.getByLabelText("GitHub token (optional for public repositories)")).toHaveAttribute(
|
||||
"type",
|
||||
"password",
|
||||
);
|
||||
expect(screen.getAllByText("Connect your repositories")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each(["github", "gitlab"])("only permits unauthenticated repository browsing for GitLab: %s", async (provider) => {
|
||||
const publicSettings = {
|
||||
...settings,
|
||||
source_provider: provider,
|
||||
gitlab_api_url: "https://gitlab.com/api/v4",
|
||||
has_github_token: false,
|
||||
has_gitlab_token: false,
|
||||
};
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) => {
|
||||
if (path === "/roi-calculator/settings") {
|
||||
return Promise.resolve(publicSettings);
|
||||
}
|
||||
if (path === "/roi-calculator/report") return Promise.resolve({ report: summary });
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Settings" }));
|
||||
const load = screen.getByRole("button", { name: "Load repositories" });
|
||||
if (provider === "github") expect(load).toBeDisabled();
|
||||
else expect(load).toBeEnabled();
|
||||
expect(screen.getByRole("textbox", { name: "Repository name" })).toBeEnabled();
|
||||
});
|
||||
|
||||
it("closes the old settings dialog when saving a different source", async () => {
|
||||
const gitlabSettings = {
|
||||
...settings,
|
||||
source_provider: "gitlab",
|
||||
gitlab_api_url: "https://gitlab.com/api/v4",
|
||||
has_gitlab_token: false,
|
||||
repos: [],
|
||||
ready: false,
|
||||
};
|
||||
vi.mocked(apiClient.put).mockResolvedValue(gitlabSettings);
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Settings" }));
|
||||
fireEvent.change(screen.getByLabelText("Repository source"), { target: { value: "gitlab" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save settings" }));
|
||||
expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeVisible();
|
||||
expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
|
||||
expect(screen.getAllByLabelText("Repository source")).toHaveLength(1);
|
||||
expect(screen.getByLabelText("Repository source")).toHaveValue("gitlab");
|
||||
});
|
||||
|
||||
it("keeps a running analysis visible when a source change finishes saving", async () => {
|
||||
const gitlabSettings = { ...settings, source_provider: "gitlab", repos: ["group/project"] };
|
||||
const runningStatus = { ...idleStatus, running: true, phase: "estimating", total: 1 };
|
||||
const saveRequest = Promise.withResolvers<typeof gitlabSettings>();
|
||||
vi.mocked(apiClient.put).mockReturnValue(saveRequest.promise);
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Settings" }));
|
||||
fireEvent.change(screen.getByLabelText("Repository source"), { target: { value: "gitlab" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save settings" }));
|
||||
vi.mocked(apiClient.get).mockResolvedValue(runningStatus);
|
||||
expect(await screen.findByRole("progressbar", { hidden: true }, { timeout: 3000 })).toBeInTheDocument();
|
||||
|
||||
saveRequest.resolve(gitlabSettings);
|
||||
await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument());
|
||||
expect(screen.getByRole("progressbar", { name: "Sync progress" })).toBeVisible();
|
||||
expect(screen.getByRole("button", { name: "Cancel sync" })).toBeEnabled();
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["success", "failure"])("ignores an old report refresh %s after switching sources", async (outcome) => {
|
||||
const gitlabSettings = { ...settings, source_provider: "gitlab", repos: [], ready: false };
|
||||
const oldRequest = Promise.withResolvers<{ report: typeof summary }>();
|
||||
const complete = { ...idleStatus, phase: "complete", finished_at: "2026-09-30T12:00:00Z" };
|
||||
vi.mocked(apiClient.put).mockResolvedValue(gitlabSettings);
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Settings" }));
|
||||
vi.mocked(apiClient.get)
|
||||
.mockClear()
|
||||
.mockImplementation((path: string) =>
|
||||
path === "/roi-calculator/report" ? oldRequest.promise : Promise.resolve(complete),
|
||||
);
|
||||
await waitFor(
|
||||
() => expect(apiClient.get).toHaveBeenCalledWith("/roi-calculator/report", { accessToken: "token" }),
|
||||
{
|
||||
timeout: 3000,
|
||||
},
|
||||
);
|
||||
fireEvent.change(screen.getByLabelText("Repository source"), { target: { value: "gitlab" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save settings" }));
|
||||
expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeVisible();
|
||||
|
||||
await act(async () => {
|
||||
if (outcome === "success") oldRequest.resolve({ report: summary });
|
||||
else oldRequest.reject(new Error("The old source is unavailable"));
|
||||
});
|
||||
expect(screen.getByRole("heading", { name: "Connect your repositories" })).toBeVisible();
|
||||
expect(screen.getByLabelText("Repository source")).toHaveValue("gitlab");
|
||||
expect(screen.queryByText("Improve request routing")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("The old source is unavailable")).not.toBeInTheDocument();
|
||||
|
||||
const nextReport = { ...summary, pulls: [{ ...summary.pulls[0], title: "New source merge request" }] };
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) =>
|
||||
Promise.resolve(
|
||||
path === "/roi-calculator/report"
|
||||
? { report: nextReport }
|
||||
: { ...complete, finished_at: "2026-09-30T13:00:00Z" },
|
||||
),
|
||||
);
|
||||
expect(await screen.findByText("New source merge request", {}, { timeout: 3000 })).toBeVisible();
|
||||
expect(screen.queryByText("Improve request routing")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("clearly identifies the sample report and returns to setup when exiting", async () => {
|
||||
|
|
@ -263,18 +381,212 @@ describe("ROICalculatorView", () => {
|
|||
fireEvent.click(await screen.findByRole("button", { name: "Preview sample report" }));
|
||||
|
||||
expect(await screen.findByText("You’re viewing demo data")).toBeVisible();
|
||||
expect(screen.getByText("Spend per estimated engineering hour")).toBeVisible();
|
||||
expect(screen.getByRole("tab", { name: "Branches" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByText("Cost / estimated hour")).toBeVisible();
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("tab", { name: "Settings" })).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Exit demo" }));
|
||||
|
||||
expect(screen.getByRole("heading", { name: "Connect GitHub to get started" })).toBeVisible();
|
||||
expect(screen.getByRole("heading", { name: "Connect your repositories" })).toBeVisible();
|
||||
expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
expect(apiClient.put).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("opens sample PR costs from a live report and restores the live data on exit", async () => {
|
||||
const samplePull = {
|
||||
...summary.pulls[0],
|
||||
title: "Sample usage breakdown",
|
||||
source_repo: "github.com/org/repo",
|
||||
source_branch: "feature/usage",
|
||||
branch_cost: {
|
||||
status: "matched",
|
||||
spend: 9.1,
|
||||
requests: 75,
|
||||
repo: "github.com/org/repo",
|
||||
branch: "feature/usage",
|
||||
},
|
||||
};
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (path === "/roi-calculator/report") {
|
||||
return Promise.resolve({
|
||||
report: options?.query?.mode === "demo" ? { ...summary, mode: "demo", pulls: [samplePull] } : summary,
|
||||
});
|
||||
}
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("tab", { name: "Branches" }));
|
||||
fireEvent.change(screen.getByRole("searchbox"), { target: { value: "no matching PR" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Preview sample report" }));
|
||||
|
||||
expect(await screen.findByText("You’re viewing demo data")).toBeVisible();
|
||||
expect(window.location.search).toBe("?demo=1");
|
||||
expect(screen.getByRole("tab", { name: "Branches" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByRole("searchbox")).toHaveValue("");
|
||||
expect(screen.getByRole("cell", { name: "$9.10" })).toBeVisible();
|
||||
expect(screen.queryByText("Improve request routing")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
|
||||
const runningStatus = { ...idleStatus, running: true, phase: "estimating", total: 1 };
|
||||
vi.mocked(apiClient.get).mockClear().mockResolvedValue(runningStatus);
|
||||
await waitFor(() => expect(apiClient.get).toHaveBeenCalledWith("/roi-calculator/sync", { accessToken: "token" }), {
|
||||
timeout: 3000,
|
||||
});
|
||||
expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Open estimate for org/repo pull request 42" }));
|
||||
expect(await screen.findByRole("dialog")).toBeVisible();
|
||||
expect(screen.getByText("75 requests")).toBeVisible();
|
||||
expect(screen.getByText(/branch:feature\/usage/)).toBeVisible();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Close" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Exit demo" }));
|
||||
|
||||
expect(screen.getByText("Improve request routing")).toBeVisible();
|
||||
expect(screen.queryByText("Sample usage breakdown")).not.toBeInTheDocument();
|
||||
expect(window.location.search).toBe("");
|
||||
expect(screen.getByRole("button", { name: "Syncing…" })).toBeDisabled();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
expect(apiClient.put).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("opens a demo link with sample data even while live analysis is running", async () => {
|
||||
window.history.replaceState(null, "", "/roi-calculator/?demo=1");
|
||||
const demoSummary = { ...summary, mode: "demo", metrics: { ...summary.metrics, total_spend: 38.4 } };
|
||||
const runningStatus = { ...idleStatus, running: true, phase: "estimating", total: 1 };
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (path === "/roi-calculator/report") {
|
||||
return Promise.resolve({ report: options?.query?.mode === "demo" ? demoSummary : summary });
|
||||
}
|
||||
return Promise.resolve(runningStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByText("You’re viewing demo data")).toBeVisible();
|
||||
expect(screen.getByText("$38.40")).toBeVisible();
|
||||
expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Exit demo" }));
|
||||
expect(screen.getByRole("progressbar")).toBeVisible();
|
||||
expect(screen.getByText("$20.00")).toBeVisible();
|
||||
expect(window.location.search).toBe("");
|
||||
});
|
||||
|
||||
it.each(["report", "sync"])("loads a demo link when the live %s request fails", async (failedRequest) => {
|
||||
window.history.replaceState(null, "", "/roi-calculator/?demo=1");
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (options?.query?.mode === "demo") return Promise.resolve({ report: { ...summary, mode: "demo" } });
|
||||
if (path === `/roi-calculator/${failedRequest}`) return Promise.reject(new Error("Live data unavailable"));
|
||||
if (path === "/roi-calculator/report") return Promise.resolve({ report: summary });
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByText("You’re viewing demo data")).toBeVisible();
|
||||
expect(screen.getByText("Gateway AI cost")).toBeVisible();
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shows the demo without waiting for a stalled live request", async () => {
|
||||
window.history.replaceState(null, "", "/roi-calculator/?demo=1");
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (options?.query?.mode === "demo") return Promise.resolve({ report: { ...summary, mode: "demo" } });
|
||||
return new Promise(() => {});
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByText("You’re viewing demo data")).toBeVisible();
|
||||
expect(screen.getByText("Gateway AI cost")).toBeVisible();
|
||||
});
|
||||
|
||||
it("keeps the live calculator usable when a demo link cannot load sample data", async () => {
|
||||
window.history.replaceState(null, "", "/roi-calculator/?demo=1&from=review#overview");
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (options?.query?.mode === "demo") return Promise.reject(new Error("Sample data unavailable"));
|
||||
if (path === "/roi-calculator/report") return Promise.resolve({ report: summary });
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByText("Gateway AI cost")).toBeVisible();
|
||||
expect(screen.getByText("$20.00")).toBeVisible();
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("Sample data unavailable");
|
||||
expect(screen.getByRole("button", { name: "Settings" })).toBeEnabled();
|
||||
expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
expect(window.location.search).toBe("?from=review");
|
||||
expect(window.location.hash).toBe("#overview");
|
||||
});
|
||||
|
||||
it.each(["report", "sync"])("waits for the live %s when exiting a demo", async (pendingRequest) => {
|
||||
window.history.replaceState(null, "", "/roi-calculator/?demo=1");
|
||||
const pending = Promise.withResolvers<{ report: typeof summary } | typeof idleStatus>();
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (options?.query?.mode === "demo") return Promise.resolve({ report: { ...summary, mode: "demo" } });
|
||||
if (path === `/roi-calculator/${pendingRequest}`) return pending.promise;
|
||||
if (path === "/roi-calculator/report") return Promise.resolve({ report: summary });
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Exit demo" }));
|
||||
expect(screen.getByText("Loading ROI Calculator…")).toBeVisible();
|
||||
expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument();
|
||||
expect(window.location.search).toBe("");
|
||||
|
||||
pending.resolve(pendingRequest === "report" ? { report: summary } : idleStatus);
|
||||
expect(await screen.findByText("Gateway AI cost")).toBeVisible();
|
||||
expect(screen.getByText("$20.00")).toBeVisible();
|
||||
expect(screen.queryByText("Loading ROI Calculator…")).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["report", "demo"])("retains a failed %s load after a successful sync poll", async (failedRequest) => {
|
||||
if (failedRequest === "demo") window.history.replaceState(null, "", "/roi-calculator/?demo=1");
|
||||
const message = `The ${failedRequest} is unavailable`;
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string, options) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (options?.query?.mode === "demo") return Promise.reject(new Error(message));
|
||||
if (path === "/roi-calculator/report") {
|
||||
return failedRequest === "report" ? Promise.reject(new Error(message)) : Promise.resolve({ report: summary });
|
||||
}
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent(message);
|
||||
const running = { ...idleStatus, running: true, phase: "estimating", total: 1 };
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) => {
|
||||
if (path === "/roi-calculator/report") return Promise.reject(new Error(message));
|
||||
return Promise.resolve(running);
|
||||
});
|
||||
expect(await screen.findByRole("progressbar", { name: "Sync progress" }, { timeout: 3000 })).toBeVisible();
|
||||
expect(screen.getByRole("alert")).toHaveTextContent(message);
|
||||
});
|
||||
|
||||
it("retries a failed initial report and clears its error only when the report recovers", async () => {
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(settings);
|
||||
if (path === "/roi-calculator/report") return Promise.reject(new Error("Report unavailable"));
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Report unavailable");
|
||||
expect(screen.getByRole("button", { name: "Settings" })).toBeEnabled();
|
||||
expect(screen.getByRole("button", { name: "Run analysis" })).toBeEnabled();
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) =>
|
||||
Promise.resolve(path === "/roi-calculator/report" ? { report: summary } : idleStatus),
|
||||
);
|
||||
expect(await screen.findByText("Gateway AI cost", {}, { timeout: 3000 })).toBeVisible();
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("returns to Overview and shows the last sync time when completion is polled from Settings", async () => {
|
||||
const runningStatus = {
|
||||
...idleStatus,
|
||||
|
|
@ -299,10 +611,9 @@ describe("ROICalculatorView", () => {
|
|||
render(<ROICalculatorView accessToken="token" />);
|
||||
|
||||
expect(await screen.findByRole("progressbar", { name: "Sync progress" })).toBeInTheDocument();
|
||||
expect(await screen.findByText("Spend per estimated engineering hour", {}, { timeout: 5000 })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("heading", { name: "Connect GitHub to get started" })).not.toBeInTheDocument();
|
||||
expect(await screen.findByText("Gateway AI cost", {}, { timeout: 5000 })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("status")).toHaveTextContent("Last synced Sep 30, 2026, 12:00 PM UTC");
|
||||
expect(screen.getByRole("status")).toHaveTextContent("57 of 57 estimates reused");
|
||||
});
|
||||
|
||||
it("shows the sync error returned by the status endpoint", async () => {
|
||||
|
|
@ -380,14 +691,14 @@ describe("ROICalculatorView", () => {
|
|||
expect(await screen.findByRole("alert", {}, { timeout: 5000 })).toHaveTextContent(
|
||||
"The sync status could not be loaded.",
|
||||
);
|
||||
expect(await screen.findByText("Spend per estimated engineering hour", {}, { timeout: 7000 })).toBeInTheDocument();
|
||||
expect(await screen.findByText("Gateway AI cost", {}, { timeout: 7000 })).toBeInTheDocument();
|
||||
expect(screen.queryByText("The sync status could not be loaded.")).not.toBeInTheDocument();
|
||||
});
|
||||
it("saves the edited schedule before running from Settings", async () => {
|
||||
vi.mocked(apiClient.put).mockResolvedValue(settings);
|
||||
vi.mocked(apiClient.post).mockResolvedValue({ ...idleStatus, running: true });
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
fireEvent.click(await screen.findByRole("tab", { name: "Settings" }));
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Settings" }));
|
||||
fireEvent.change(screen.getByLabelText("Update interval (hours)"), { target: { value: "6" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save and run analysis" }));
|
||||
await waitFor(() => expect(apiClient.post).toHaveBeenCalledWith("/roi-calculator/sync", { accessToken: "token" }));
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
"use client";
|
||||
|
||||
import { Page } from "@/components/shared/Page";
|
||||
import { Page, PageTabsList, PageTabsTrigger } from "@/components/shared/Page";
|
||||
import React from "react";
|
||||
import { Calculator, RefreshCw } from "lucide-react";
|
||||
import { Calculator, RefreshCw, Settings2 } from "lucide-react";
|
||||
|
||||
import { apiClient } from "@/components/networking";
|
||||
import { DemoNotice } from "@/components/shared/DemoNotice";
|
||||
|
|
@ -11,12 +11,13 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
|||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Tabs } from "@/components/ui/tabs";
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog";
|
||||
import { extractErrorMessage } from "@/utils/errorUtils";
|
||||
import { isProxyAdminTierRole } from "@/utils/roles";
|
||||
import ROISettingsPanel from "./ROISettingsPanel";
|
||||
import { IdentityMatchDialog, type PersonMatchSelection, PullReasoningDialog } from "./ROICalculatorDialogs";
|
||||
import { ROIOverview, ROIPeopleView } from "./ROICalculatorViews";
|
||||
import { ROIBranches, ROIOverview, ROIPeopleView } from "./ROICalculatorViews";
|
||||
import { filterPulls, formatSyncedAt } from "./roiCalculatorData";
|
||||
import type {
|
||||
ROIIdentityMapResponse,
|
||||
|
|
@ -28,7 +29,7 @@ import type {
|
|||
ROISyncStatus,
|
||||
} from "./roiCalculatorData";
|
||||
|
||||
type View = "overview" | "people" | "settings";
|
||||
type View = "overview" | "people" | "branches";
|
||||
|
||||
const IDLE_STATUS: ROISyncStatus = {
|
||||
running: false,
|
||||
|
|
@ -43,6 +44,13 @@ const IDLE_STATUS: ROISyncStatus = {
|
|||
error: null,
|
||||
};
|
||||
|
||||
function updateDemoUrl(enabled: boolean) {
|
||||
const url = new URL(window.location.href);
|
||||
if (enabled) url.searchParams.set("demo", "1");
|
||||
else url.searchParams.delete("demo");
|
||||
window.history.replaceState(null, "", url);
|
||||
}
|
||||
|
||||
export default function ROICalculatorView({
|
||||
accessToken,
|
||||
userRole = null,
|
||||
|
|
@ -55,6 +63,7 @@ export default function ROICalculatorView({
|
|||
const [sampleSummary, setSampleSummary] = React.useState<ROISummary | null>(null);
|
||||
const adminReadOnly = isViewOnly && isProxyAdminTierRole(userRole ?? "");
|
||||
const readOnly = adminReadOnly || sampleSummary !== null;
|
||||
const [settingsOpen, setSettingsOpen] = React.useState(false);
|
||||
const [view, setView] = React.useState<View>("overview");
|
||||
const [settings, setSettings] = React.useState<ROISettings | null>(null);
|
||||
const [liveSummary, setSummary] = React.useState<ROISummary | null>(null);
|
||||
|
|
@ -63,8 +72,16 @@ export default function ROICalculatorView({
|
|||
const [selectedPull, setSelectedPull] = React.useState<ROIPull | null>(null);
|
||||
const [matchingPerson, setMatchingPerson] = React.useState<PersonMatchSelection | null>(null);
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
const [demoError, setDemoError] = React.useState<string | null>(null);
|
||||
const [reportError, setReportError] = React.useState<string | null>(null);
|
||||
const [syncError, setSyncError] = React.useState<string | null>(null);
|
||||
const [loadingInitialData, setLoadingInitialData] = React.useState(true);
|
||||
const [loadingLiveData, setLoadingLiveData] = React.useState(true);
|
||||
const statusRef = React.useRef<ROISyncStatus>(IDLE_STATUS);
|
||||
const settingsLoaded = settings !== null;
|
||||
const reportNeedsRefresh = React.useRef(false);
|
||||
const sourceRevision = React.useRef(0);
|
||||
const settingsLoaded = settings !== null && !loadingInitialData && !loadingLiveData;
|
||||
const requestError = [error, demoError, reportError, syncError].filter(Boolean).join(" ");
|
||||
const [query, setQuery] = React.useState("");
|
||||
|
||||
const loadReport = React.useCallback(async () => {
|
||||
|
|
@ -76,21 +93,63 @@ export default function ROICalculatorView({
|
|||
React.useEffect(() => {
|
||||
if (!accessToken) return;
|
||||
let cancelled = false;
|
||||
Promise.all([
|
||||
apiClient.get<ROISettings>("/roi-calculator/settings", { accessToken }),
|
||||
apiClient.get<ROIReportResponse>("/roi-calculator/report", { accessToken }),
|
||||
apiClient.get<ROISyncStatus>("/roi-calculator/sync", { accessToken }),
|
||||
])
|
||||
.then(([nextSettings, reportResponse, syncStatus]) => {
|
||||
const demoRequested = new URLSearchParams(window.location.search).get("demo") === "1";
|
||||
const settingsRequest = apiClient.get<ROISettings>("/roi-calculator/settings", { accessToken });
|
||||
const reportRequest = apiClient
|
||||
.get<ROIReportResponse>("/roi-calculator/report", { accessToken })
|
||||
.then((response) => {
|
||||
if (cancelled) return;
|
||||
setSummary(response.report);
|
||||
setReportError(null);
|
||||
reportNeedsRefresh.current = false;
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (cancelled) return;
|
||||
setReportError(extractErrorMessage(reason));
|
||||
reportNeedsRefresh.current = true;
|
||||
});
|
||||
const statusRequest = apiClient
|
||||
.get<ROISyncStatus>("/roi-calculator/sync", { accessToken })
|
||||
.then((syncStatus) => {
|
||||
if (cancelled) return;
|
||||
setSettings(nextSettings);
|
||||
setSummary(reportResponse.report);
|
||||
setStatus(syncStatus);
|
||||
statusRef.current = syncStatus;
|
||||
setSyncError(null);
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (!cancelled) setSyncError(extractErrorMessage(reason));
|
||||
});
|
||||
const liveData = Promise.all([reportRequest, statusRequest])
|
||||
.then(() => null)
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoadingLiveData(false);
|
||||
});
|
||||
Promise.all([
|
||||
settingsRequest,
|
||||
demoRequested
|
||||
? apiClient
|
||||
.get<ROIReportResponse>("/roi-calculator/report", { accessToken, query: { mode: "demo" } })
|
||||
.catch((reason: unknown) => {
|
||||
if (!cancelled) {
|
||||
setDemoError(`Could not load demo data: ${extractErrorMessage(reason)}`);
|
||||
updateDemoUrl(false);
|
||||
}
|
||||
return liveData;
|
||||
})
|
||||
: liveData,
|
||||
])
|
||||
.then(([nextSettings, sampleResponse]) => {
|
||||
if (cancelled) return;
|
||||
setSettings(nextSettings);
|
||||
setSampleSummary(sampleResponse?.report ?? null);
|
||||
setError(null);
|
||||
if (sampleResponse) setDemoError(null);
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (!cancelled) setError(extractErrorMessage(reason));
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoadingInitialData(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
|
|
@ -101,31 +160,36 @@ export default function ROICalculatorView({
|
|||
if (!accessToken || !settingsLoaded) return;
|
||||
let cancelled = false;
|
||||
let requestInFlight = false;
|
||||
let reportNeedsRefresh = false;
|
||||
const interval = window.setInterval(() => {
|
||||
if (requestInFlight) return;
|
||||
requestInFlight = true;
|
||||
const revision = sourceRevision.current;
|
||||
const isCurrent = () => !cancelled && revision === sourceRevision.current;
|
||||
apiClient
|
||||
.get<ROISyncStatus>("/roi-calculator/sync", { accessToken })
|
||||
.then(async (nextStatus) => {
|
||||
if (cancelled) return;
|
||||
if (!isCurrent()) return;
|
||||
const previousStatus = statusRef.current;
|
||||
statusRef.current = nextStatus;
|
||||
setStatus(nextStatus);
|
||||
setSyncError(null);
|
||||
const finished = !nextStatus.running && nextStatus.phase === "complete";
|
||||
const reportChanged = previousStatus.running || nextStatus.finished_at !== previousStatus.finished_at;
|
||||
if (finished && (reportChanged || reportNeedsRefresh)) {
|
||||
reportNeedsRefresh = true;
|
||||
const report = await loadReport();
|
||||
if (cancelled) return;
|
||||
setSummary(report);
|
||||
reportNeedsRefresh = false;
|
||||
setView((current) => (current === "settings" ? "overview" : current));
|
||||
if (reportNeedsRefresh.current || (finished && reportChanged)) {
|
||||
reportNeedsRefresh.current = true;
|
||||
try {
|
||||
const report = await loadReport();
|
||||
if (!isCurrent()) return;
|
||||
setSummary(report);
|
||||
setReportError(null);
|
||||
reportNeedsRefresh.current = false;
|
||||
} catch (reason: unknown) {
|
||||
if (isCurrent()) setReportError(extractErrorMessage(reason));
|
||||
}
|
||||
}
|
||||
if (!cancelled) setError(null);
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (!cancelled) setError(extractErrorMessage(reason));
|
||||
if (isCurrent()) setSyncError(extractErrorMessage(reason));
|
||||
})
|
||||
.finally(() => {
|
||||
requestInFlight = false;
|
||||
|
|
@ -144,6 +208,8 @@ export default function ROICalculatorView({
|
|||
const nextStatus = await apiClient.post<ROISyncStatus>("/roi-calculator/sync", { accessToken });
|
||||
statusRef.current = nextStatus;
|
||||
setStatus(nextStatus);
|
||||
setSyncError(null);
|
||||
setSettingsOpen(false);
|
||||
} catch (reason) {
|
||||
setError(extractErrorMessage(reason));
|
||||
}
|
||||
|
|
@ -152,7 +218,11 @@ export default function ROICalculatorView({
|
|||
const cancelSync = React.useCallback(async () => {
|
||||
if (!accessToken || readOnly) return;
|
||||
try {
|
||||
setStatus(await apiClient.delete<ROISyncStatus>("/roi-calculator/sync", { accessToken }));
|
||||
const nextStatus = await apiClient.delete<ROISyncStatus>("/roi-calculator/sync", { accessToken });
|
||||
setStatus(nextStatus);
|
||||
statusRef.current = nextStatus;
|
||||
setSyncError(null);
|
||||
setError(null);
|
||||
} catch (reason) {
|
||||
setError(extractErrorMessage(reason));
|
||||
}
|
||||
|
|
@ -166,6 +236,8 @@ export default function ROICalculatorView({
|
|||
body: payload,
|
||||
});
|
||||
setSummary(response.report);
|
||||
setReportError(null);
|
||||
reportNeedsRefresh.current = false;
|
||||
setSettings((current) => (current ? { ...current, identity_map: response.identity_map } : current));
|
||||
},
|
||||
[accessToken, readOnly],
|
||||
|
|
@ -173,7 +245,7 @@ export default function ROICalculatorView({
|
|||
|
||||
const filteredPulls = React.useMemo(() => (summary ? filterPulls(summary.pulls, query) : []), [query, summary]);
|
||||
|
||||
if (error && !settings) {
|
||||
if (error && !settings && !loadingInitialData) {
|
||||
return (
|
||||
<div className="p-8">
|
||||
<Alert variant="destructive">
|
||||
|
|
@ -184,9 +256,11 @@ export default function ROICalculatorView({
|
|||
);
|
||||
}
|
||||
|
||||
if (!settings) {
|
||||
const awaitingLiveData = !sampleSummary && loadingLiveData;
|
||||
if (!settings || loadingInitialData || awaitingLiveData) {
|
||||
return (
|
||||
<div className="space-y-6 p-8">
|
||||
<p role="status">Loading ROI Calculator…</p>
|
||||
<Skeleton className="h-16 w-96" />
|
||||
<Skeleton className="h-96 w-full" />
|
||||
</div>
|
||||
|
|
@ -200,58 +274,86 @@ export default function ROICalculatorView({
|
|||
query: { mode: "demo" },
|
||||
});
|
||||
setSampleSummary(response.report);
|
||||
setView("overview");
|
||||
setDemoError(null);
|
||||
updateDemoUrl(true);
|
||||
setView("branches");
|
||||
setQuery("");
|
||||
} catch (reason) {
|
||||
setError(extractErrorMessage(reason));
|
||||
setDemoError(`Could not load demo data: ${extractErrorMessage(reason)}`);
|
||||
}
|
||||
};
|
||||
const resetView = (updated: ROISettings) => {
|
||||
const resetView = (updated: ROISettings, resetSyncStatus = true) => {
|
||||
sourceRevision.current += 1;
|
||||
setSettings(updated);
|
||||
setSummary(null);
|
||||
setReportError(null);
|
||||
reportNeedsRefresh.current = false;
|
||||
setSettingsOpen(false);
|
||||
setView("overview");
|
||||
setStatus(IDLE_STATUS);
|
||||
statusRef.current = IDLE_STATUS;
|
||||
if (resetSyncStatus) {
|
||||
setStatus(IDLE_STATUS);
|
||||
statusRef.current = IDLE_STATUS;
|
||||
setSyncError(null);
|
||||
setError(null);
|
||||
}
|
||||
};
|
||||
const showLiveStatus = !sampleSummary && !status.running;
|
||||
const scheduleLabel = settings.update_interval_minutes ? "Automatic updates enabled" : "Manual updates";
|
||||
const showReportActions = summary !== null || reportError !== null;
|
||||
const progress = status.total > 0 ? Math.min(100, (status.done / status.total) * 100) : 0;
|
||||
const statusIsIdleOrComplete = status.phase === "idle" || status.phase === "complete";
|
||||
const syncIsUpToDate = !status.running && statusIsIdleOrComplete;
|
||||
const syncedAt = syncIsUpToDate ? summary?.synced_at : null;
|
||||
const syncedAt = sampleSummary?.synced_at ?? (syncIsUpToDate ? summary?.synced_at : null);
|
||||
|
||||
return (
|
||||
<Page>
|
||||
<PageHeader>
|
||||
<Page className="gap-6">
|
||||
<PageHeader className="space-y-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<PageHeaderTitle>
|
||||
<Calculator />
|
||||
ROI Calculator
|
||||
</PageHeaderTitle>
|
||||
{!liveSummary && showLiveStatus && (
|
||||
<Button variant="outline" onClick={() => void previewSample()}>
|
||||
Preview sample report
|
||||
</Button>
|
||||
{!sampleSummary && (
|
||||
<div className="flex items-center gap-2">
|
||||
{showLiveStatus && (
|
||||
<Button variant="ghost" onClick={() => void previewSample()}>
|
||||
Preview sample report
|
||||
</Button>
|
||||
)}
|
||||
{showReportActions && (
|
||||
<Button variant="outline" onClick={() => setSettingsOpen(true)}>
|
||||
<Settings2 />
|
||||
Settings
|
||||
</Button>
|
||||
)}
|
||||
{showReportActions && !readOnly && (
|
||||
<Button onClick={() => void startSync()} disabled={status.running || !settings.ready}>
|
||||
<RefreshCw className={status.running ? "animate-spin" : ""} />
|
||||
{status.running ? "Syncing…" : "Run analysis"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<PageHeaderDescription>
|
||||
{summary
|
||||
? `${summary.start} through ${summary.end} · UTC`
|
||||
: "Compare gateway spend with estimated engineering effort for merged pull requests"}
|
||||
<PageHeaderDescription className="flex flex-wrap items-center gap-x-4 gap-y-1">
|
||||
<span>
|
||||
{summary
|
||||
? `${summary.start} through ${summary.end} · UTC`
|
||||
: "Compare AI costs with estimated engineering effort"}
|
||||
</span>
|
||||
{syncedAt && (
|
||||
<span className="mt-1 block text-xs text-muted-foreground" role="status">
|
||||
<span className="text-xs" role="status">
|
||||
Last synced {formatSyncedAt(syncedAt)}
|
||||
{!status.running && status.phase === "complete" && status.reused > 0
|
||||
? ` · ${status.reused} of ${status.total} estimates reused`
|
||||
: ""}
|
||||
</span>
|
||||
)}
|
||||
</PageHeaderDescription>
|
||||
</PageHeader>
|
||||
{sampleSummary && <DemoNotice onExit={() => setSampleSummary(null)} />}
|
||||
{liveSummary && showLiveStatus && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{status.next_update ? `Next update ${formatSyncedAt(status.next_update)}` : scheduleLabel}
|
||||
</p>
|
||||
{sampleSummary && (
|
||||
<DemoNotice
|
||||
onExit={() => {
|
||||
updateDemoUrl(false);
|
||||
setSampleSummary(null);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{adminReadOnly && (
|
||||
<p className="text-sm text-muted-foreground" role="note">
|
||||
|
|
@ -260,30 +362,22 @@ export default function ROICalculatorView({
|
|||
)}
|
||||
|
||||
{summary && (
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<Tabs value={view} onValueChange={(value) => setView(value as View)}>
|
||||
<TabsList aria-label="ROI Calculator views">
|
||||
<TabsTrigger value="overview">Overview</TabsTrigger>
|
||||
<TabsTrigger value="people">People</TabsTrigger>
|
||||
{!sampleSummary && <TabsTrigger value="settings">Settings</TabsTrigger>}
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
{view !== "settings" && !readOnly && (
|
||||
<Button onClick={() => void startSync()} disabled={status.running || !settings.ready}>
|
||||
<RefreshCw className={status.running ? "animate-spin" : ""} />
|
||||
{status.running ? "Syncing…" : "Run analysis"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
<Tabs value={view} onValueChange={(value) => setView(value as View)}>
|
||||
<PageTabsList aria-label="ROI Calculator views">
|
||||
<PageTabsTrigger value="overview">Overview</PageTabsTrigger>
|
||||
<PageTabsTrigger value="people">People</PageTabsTrigger>
|
||||
<PageTabsTrigger value="branches">Branches</PageTabsTrigger>
|
||||
</PageTabsList>
|
||||
</Tabs>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
{!sampleSummary && requestError && (
|
||||
<Alert variant="destructive">
|
||||
<AlertTitle>ROI Calculator request failed</AlertTitle>
|
||||
<AlertDescription>{error}</AlertDescription>
|
||||
<AlertDescription>{requestError}</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
{status.error && (
|
||||
{!sampleSummary && status.error && (
|
||||
<Alert variant="destructive">
|
||||
<AlertTitle>Sync failed</AlertTitle>
|
||||
<AlertDescription>{status.error}</AlertDescription>
|
||||
|
|
@ -295,9 +389,9 @@ export default function ROICalculatorView({
|
|||
<AlertDescription>{warning}</AlertDescription>
|
||||
</Alert>
|
||||
))}
|
||||
{status.running && (
|
||||
{!sampleSummary && status.running && (
|
||||
<Card>
|
||||
<CardContent className="flex flex-wrap items-center justify-between gap-4 pt-6">
|
||||
<CardContent className="flex flex-wrap items-center justify-between gap-4">
|
||||
<div
|
||||
aria-label="Sync progress"
|
||||
aria-valuemax={100}
|
||||
|
|
@ -311,7 +405,7 @@ export default function ROICalculatorView({
|
|||
<div className="h-full bg-primary transition-all" style={{ width: `${progress}%` }} />
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{status.done} of {status.total} pull requests processed · {status.reused} reused
|
||||
{status.done} of {status.total} changes processed · {status.reused} reused
|
||||
{` · ${status.elapsed_seconds ?? 0}s elapsed`}
|
||||
{status.remaining_seconds != null ? ` · about ${status.remaining_seconds}s remaining` : ""}
|
||||
</p>
|
||||
|
|
@ -325,7 +419,7 @@ export default function ROICalculatorView({
|
|||
</Card>
|
||||
)}
|
||||
|
||||
{view === "settings" || (!summary && !status.running) ? (
|
||||
{!summary && !status.running && !reportError ? (
|
||||
<ROISettingsPanel
|
||||
accessToken={accessToken}
|
||||
initialSettings={settings}
|
||||
|
|
@ -339,12 +433,19 @@ export default function ROICalculatorView({
|
|||
) : null}
|
||||
{view === "overview" && summary && (
|
||||
<ROIOverview
|
||||
summary={summary}
|
||||
onSelectPull={setSelectedPull}
|
||||
onViewPeople={() => setView("people")}
|
||||
onViewBranches={() => setView("branches")}
|
||||
/>
|
||||
)}
|
||||
{view === "branches" && summary && (
|
||||
<ROIBranches
|
||||
summary={summary}
|
||||
pulls={filteredPulls}
|
||||
query={query}
|
||||
onQueryChange={setQuery}
|
||||
onSelectPull={setSelectedPull}
|
||||
onViewPeople={() => setView("people")}
|
||||
/>
|
||||
)}
|
||||
{view === "people" && summary && (
|
||||
|
|
@ -355,6 +456,34 @@ export default function ROICalculatorView({
|
|||
readOnly={readOnly}
|
||||
/>
|
||||
)}
|
||||
<Dialog open={settingsOpen} onOpenChange={setSettingsOpen}>
|
||||
<DialogContent className="max-h-[90dvh] overflow-y-auto sm:max-w-2xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Calculator settings</DialogTitle>
|
||||
<DialogDescription>Connect repositories and choose how to estimate effort.</DialogDescription>
|
||||
</DialogHeader>
|
||||
<ROISettingsPanel
|
||||
accessToken={accessToken}
|
||||
initialSettings={settings}
|
||||
onboarding={false}
|
||||
onSaved={(updated) => {
|
||||
if (
|
||||
updated.source_provider !== settings.source_provider ||
|
||||
updated.github_api_url !== settings.github_api_url ||
|
||||
updated.gitlab_api_url !== settings.gitlab_api_url
|
||||
) {
|
||||
resetView(updated, false);
|
||||
return;
|
||||
}
|
||||
setSettings(updated);
|
||||
}}
|
||||
onReset={resetView}
|
||||
onStartSync={startSync}
|
||||
readOnly={readOnly}
|
||||
syncDisabled={status.running}
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
<PullReasoningDialog pull={selectedPull} summary={summary} onClose={() => setSelectedPull(null)} />
|
||||
{!readOnly && (
|
||||
<IdentityMatchDialog
|
||||
|
|
|
|||
|
|
@ -1,175 +1,274 @@
|
|||
"use client";
|
||||
|
||||
import React from "react";
|
||||
import { Bar, CartesianGrid, ComposedChart, Line, XAxis, YAxis } from "recharts";
|
||||
|
||||
import { ChevronDown, Download, GitBranch, Search } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import {
|
||||
ChartContainer,
|
||||
ChartLegend,
|
||||
ChartLegendContent,
|
||||
ChartTooltip,
|
||||
ChartTooltipContent,
|
||||
} from "@/components/ui/chart";
|
||||
import type { ChartConfig } from "@/components/ui/chart";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { coverageLabel, peopleCsv, effortNote, estimateLabel, formatMoney, formatNumber } from "./roiCalculatorData";
|
||||
import {
|
||||
peopleCsv,
|
||||
effortNote,
|
||||
estimateLabel,
|
||||
formatMoney,
|
||||
formatNumber,
|
||||
branchCostLabel,
|
||||
highestCostPulls,
|
||||
} from "./roiCalculatorData";
|
||||
import type { ROIPerson, ROIPull, ROISummary } from "./roiCalculatorData";
|
||||
|
||||
const CHART_CONFIG = {
|
||||
spend: { label: "Matched spend", color: "var(--chart-1)" },
|
||||
hours: { label: "Estimated hours", color: "var(--chart-2)" },
|
||||
} satisfies ChartConfig;
|
||||
type PullSelection = { summary: ROISummary; onSelectPull: (pull: ROIPull) => void };
|
||||
|
||||
export function ROIOverview({
|
||||
summary,
|
||||
pulls,
|
||||
query,
|
||||
onQueryChange,
|
||||
onSelectPull,
|
||||
onViewPeople,
|
||||
}: {
|
||||
summary: ROISummary;
|
||||
pulls: ROIPull[];
|
||||
query: string;
|
||||
onQueryChange: (value: string) => void;
|
||||
onSelectPull: (pull: ROIPull) => void;
|
||||
onViewBranches,
|
||||
}: PullSelection & {
|
||||
onViewPeople: () => void;
|
||||
onViewBranches: () => void;
|
||||
}) {
|
||||
const [pagination, setPagination] = React.useState({ query, visibleCount: 10 });
|
||||
const visibleCount = pagination.query === query ? pagination.visibleCount : 10;
|
||||
const metrics = summary.metrics;
|
||||
const unavailableRate =
|
||||
metrics.output_hours > 0
|
||||
? "Spend per estimated hour is unavailable until all selected repositories can be read."
|
||||
: "A rate requires matched estimated hours greater than zero and access to all selected repositories.";
|
||||
const branches = summary.branch_metrics;
|
||||
const topPulls = highestCostPulls(summary.pulls);
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<section aria-label="Spend and estimated engineering effort" className="grid gap-4 md:grid-cols-2 xl:grid-cols-4">
|
||||
<MetricCard title="Spend per estimated engineering hour" value={formatMoney(metrics.cost_per_hour)} />
|
||||
<MetricCard title="Matched gateway spend" value={formatMoney(metrics.matched_spend)} />
|
||||
<MetricCard title="Estimated engineering hours" value={`${formatNumber(metrics.output_hours)} hrs`} />
|
||||
<MetricCard title="PR email coverage" value={coverageLabel(summary)} />
|
||||
</section>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{formatMoney(metrics.excluded_spend)} of {formatMoney(metrics.total_spend)} total gateway spend is excluded from
|
||||
the matched cohort.
|
||||
</p>
|
||||
<details className="rounded-lg border p-4 text-sm">
|
||||
<summary className="cursor-pointer font-medium">Calculation details</summary>
|
||||
<div className="space-y-3 pt-3 text-muted-foreground">
|
||||
<p>
|
||||
{metrics.cost_per_hour != null
|
||||
? `${formatMoney(metrics.matched_spend)} gateway spend ÷ ${formatNumber(metrics.output_hours)} estimated engineering hours = ${formatMoney(metrics.cost_per_hour)} per estimated hour.`
|
||||
: unavailableRate}
|
||||
</p>
|
||||
<p>
|
||||
The comparison includes {metrics.cohort_people} matched {metrics.cohort_people === 1 ? "person" : "people"}{" "}
|
||||
with complete PR estimates, for the same period in UTC. {metrics.matched_prs} of {metrics.merged_prs} PRs
|
||||
have email matches. {formatMoney(metrics.excluded_spend)} of {formatMoney(metrics.total_spend)} total
|
||||
gateway spend is excluded.
|
||||
</p>
|
||||
<p>
|
||||
Gateway spend includes all of each person’s usage, across repositories. This does not measure hours saved by
|
||||
AI or financial returns.
|
||||
</p>
|
||||
<Button variant="link" className="h-auto p-0" onClick={onViewPeople}>
|
||||
Review email matches
|
||||
</Button>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Spend and estimated engineering effort</CardTitle>
|
||||
<CardDescription>
|
||||
Daily matched gateway spend and estimated engineering hours for the same UTC period
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<ChartContainer config={CHART_CONFIG} className="h-[320px] w-full">
|
||||
<ComposedChart data={summary.trend} margin={{ left: 8, right: 8 }}>
|
||||
<CartesianGrid vertical={false} />
|
||||
<XAxis dataKey="date" tickLine={false} axisLine={false} minTickGap={36} />
|
||||
<YAxis yAxisId="spend" tickFormatter={(value) => formatMoney(Number(value))} />
|
||||
<YAxis yAxisId="hours" orientation="right" domain={[0, "auto"]} />
|
||||
<ChartTooltip content={<ChartTooltipContent />} />
|
||||
<ChartLegend content={<ChartLegendContent />} />
|
||||
<Bar yAxisId="spend" dataKey="spend" fill="var(--color-spend)" isAnimationActive={false} />
|
||||
<Line
|
||||
yAxisId="hours"
|
||||
dataKey="hours"
|
||||
stroke="var(--color-hours)"
|
||||
strokeWidth={2}
|
||||
dot={false}
|
||||
isAnimationActive={false}
|
||||
/>
|
||||
</ComposedChart>
|
||||
</ChartContainer>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader className="flex-row flex-wrap items-center justify-between gap-4">
|
||||
<div>
|
||||
<CardTitle>Pull requests</CardTitle>
|
||||
<CardDescription>
|
||||
{metrics.merged_prs} merged · {metrics.estimated_prs} estimated · {metrics.pending_prs} need attention
|
||||
</CardDescription>
|
||||
</div>
|
||||
<Input
|
||||
aria-label="Search pull requests"
|
||||
className="w-full sm:max-w-xs"
|
||||
placeholder="Search pull requests"
|
||||
type="search"
|
||||
value={query}
|
||||
onChange={(event) => onQueryChange(event.target.value)}
|
||||
<div className="space-y-8">
|
||||
<div className="@container overflow-hidden rounded-xl border">
|
||||
<dl aria-label="Report overview" className="grid grid-cols-2 @min-[760px]:grid-cols-4">
|
||||
<MetricCard
|
||||
title="Gateway AI cost"
|
||||
value={formatMoney(metrics.total_spend)}
|
||||
description="All gateway usage in this period"
|
||||
primary
|
||||
/>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Pull request</TableHead>
|
||||
<TableHead className="text-right">Estimated hours</TableHead>
|
||||
<MetricCard
|
||||
title="Estimated effort"
|
||||
value={`${formatNumber(metrics.total_output_hours)} hrs`}
|
||||
description={summary.effort_basis === "without_ai" ? "Estimated without AI" : "Check estimate assumptions"}
|
||||
/>
|
||||
<MetricCard
|
||||
title="Merged changes"
|
||||
value={formatNumber(metrics.merged_prs)}
|
||||
description={`${metrics.estimated_prs} estimated`}
|
||||
/>
|
||||
<MetricCard
|
||||
title="People"
|
||||
value={formatNumber(metrics.people_with_prs)}
|
||||
description="Contributors to merged work"
|
||||
/>
|
||||
</dl>
|
||||
</div>
|
||||
<section aria-label="Cost coverage" className="space-y-4">
|
||||
<div className="space-y-1">
|
||||
<h2 className="text-base font-semibold">Where AI costs are matched</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
People use gateway account costs. Branches use tagged requests for these repositories.
|
||||
</p>
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-xl border">
|
||||
<Table className="min-w-[600px]">
|
||||
<TableHeader className="bg-muted/40">
|
||||
<TableRow className="hover:bg-transparent">
|
||||
<TableHead className="px-4 text-xs">View</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs">Matched cost</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs">Unmatched cost</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs">Changes matched</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{pulls.slice(0, visibleCount).map((pull) => (
|
||||
<TableRow key={`${pull.repo}#${pull.number}`}>
|
||||
<TableCell>
|
||||
<Button
|
||||
aria-label={`Open estimate for ${pull.repo} pull request ${pull.number}`}
|
||||
className="h-auto whitespace-normal p-0 text-left"
|
||||
variant="link"
|
||||
onClick={() => onSelectPull(pull)}
|
||||
>
|
||||
<span>
|
||||
<span className="block font-medium">{pull.title}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{pull.repo} #{pull.number} · {pull.login}
|
||||
</span>
|
||||
</span>
|
||||
</Button>
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{estimateLabel(pull.estimate)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{pulls.length === 0 && (
|
||||
<TableRow>
|
||||
<TableCell className="text-center text-muted-foreground" colSpan={2}>
|
||||
{query ? "No matching pull requests." : "No merged pull requests in this period."}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
<TableRow>
|
||||
<TableCell className="px-4 py-3">
|
||||
<Button variant="link" className="h-auto p-0" onClick={onViewPeople}>
|
||||
People
|
||||
</Button>
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right font-medium tabular-nums">
|
||||
{formatMoney(metrics.matched_spend)}
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right text-muted-foreground tabular-nums">
|
||||
{formatMoney(metrics.excluded_spend)}
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right tabular-nums">
|
||||
{metrics.matched_prs} / {metrics.merged_prs}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell className="px-4 py-3">
|
||||
<Button variant="link" className="h-auto p-0" onClick={onViewBranches}>
|
||||
Branches
|
||||
</Button>
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right font-medium tabular-nums">
|
||||
{formatMoney(branches?.spend)}
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right text-muted-foreground tabular-nums">
|
||||
{formatMoney(branches?.unlinked_spend)}
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right tabular-nums">
|
||||
{branches?.matched_pulls ?? 0} / {metrics.merged_prs}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableBody>
|
||||
</Table>
|
||||
{pulls.length > visibleCount && (
|
||||
</div>
|
||||
</section>
|
||||
<ROIPulls
|
||||
summary={summary}
|
||||
pulls={topPulls}
|
||||
onSelectPull={onSelectPull}
|
||||
compact
|
||||
onViewBranches={onViewBranches}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ROIBranches({
|
||||
summary,
|
||||
onSelectPull,
|
||||
pulls,
|
||||
query,
|
||||
onQueryChange,
|
||||
}: PullSelection & {
|
||||
pulls: ROIPull[];
|
||||
query: string;
|
||||
onQueryChange: (query: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<section aria-label="Branch cost analysis" className="space-y-4">
|
||||
<div className="@container overflow-hidden rounded-xl border">
|
||||
<ROIMetrics summary={summary} branchMode />
|
||||
<ROIComparison summary={summary} branchMode />
|
||||
</div>
|
||||
</section>
|
||||
<ROIPulls
|
||||
summary={summary}
|
||||
pulls={pulls}
|
||||
query={query}
|
||||
onQueryChange={onQueryChange}
|
||||
onSelectPull={onSelectPull}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ROIPulls({
|
||||
summary,
|
||||
onSelectPull,
|
||||
pulls,
|
||||
query = "",
|
||||
onQueryChange,
|
||||
compact = false,
|
||||
onViewBranches,
|
||||
}: PullSelection & {
|
||||
pulls: ROIPull[];
|
||||
query?: string;
|
||||
onQueryChange?: (query: string) => void;
|
||||
compact?: boolean;
|
||||
onViewBranches?: () => void;
|
||||
}) {
|
||||
const changeName = summary.source_provider === "gitlab" ? "merge request" : "pull request";
|
||||
const [pagination, setPagination] = React.useState({ query, visibleCount: 10 });
|
||||
const visibleCount = pagination.query === query ? pagination.visibleCount : 10;
|
||||
const metrics = summary.metrics;
|
||||
const emptyMessage = compact
|
||||
? "No merged changes with tagged costs yet. Open Branches to see how to add tags."
|
||||
: `No merged ${changeName}s in this period.`;
|
||||
return (
|
||||
<section aria-label={`Merged ${changeName}s`} className="space-y-4">
|
||||
<div className="flex flex-wrap items-center justify-between gap-4">
|
||||
<div className="space-y-1">
|
||||
<h2 className="text-base font-semibold">{compact ? "Highest-cost changes" : "Costs by branch"}</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{compact
|
||||
? "Merged work ranked by tagged AI cost"
|
||||
: `${metrics.merged_prs} ${changeName}s · ${metrics.estimated_prs} estimated`}
|
||||
{!compact && metrics.pending_prs > 0 && (
|
||||
<span className="text-amber-700 dark:text-amber-400"> · {metrics.pending_prs} need attention</span>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
{compact ? (
|
||||
<Button variant="outline" onClick={onViewBranches}>
|
||||
View all branches
|
||||
</Button>
|
||||
) : (
|
||||
<div className="relative w-full sm:w-64">
|
||||
<Search
|
||||
aria-hidden="true"
|
||||
className="pointer-events-none absolute left-3 top-1/2 size-4 -translate-y-1/2 text-muted-foreground"
|
||||
/>
|
||||
<Input
|
||||
aria-label={`Search ${changeName}s`}
|
||||
className="pl-9"
|
||||
placeholder={`Search ${changeName}s`}
|
||||
type="search"
|
||||
value={query}
|
||||
onChange={(event) => onQueryChange?.(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-xl border">
|
||||
<Table className="min-w-[600px] table-fixed">
|
||||
<TableHeader className="bg-muted/40">
|
||||
<TableRow className="hover:bg-transparent">
|
||||
<TableHead className="w-3/5 px-4 text-xs text-muted-foreground">
|
||||
{changeName === "merge request" ? "Merge request" : "Pull request"}
|
||||
</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs text-muted-foreground">AI cost</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs text-muted-foreground">Estimated effort</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{pulls.slice(0, visibleCount).map((pull) => (
|
||||
<TableRow key={`${pull.repo}#${pull.number}`}>
|
||||
<TableCell className="whitespace-normal px-4 py-3">
|
||||
<Button
|
||||
aria-label={`Open estimate for ${pull.repo} ${changeName} ${pull.number}`}
|
||||
className="h-auto w-full justify-start whitespace-normal p-0 text-left"
|
||||
variant="link"
|
||||
onClick={() => onSelectPull(pull)}
|
||||
>
|
||||
<span className="min-w-0 space-y-1">
|
||||
<span className="block break-words font-medium leading-5">{pull.title}</span>
|
||||
<span className="block break-all text-xs font-normal text-muted-foreground">
|
||||
{pull.repo} #{pull.number} · {pull.login}
|
||||
</span>
|
||||
{!compact && pull.source_branch && (
|
||||
<span className="flex items-start gap-1.5 text-xs font-normal text-muted-foreground">
|
||||
<GitBranch aria-hidden="true" className="mt-0.5 size-3 shrink-0" />
|
||||
<span className="break-all">{pull.source_branch}</span>
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</Button>
|
||||
</TableCell>
|
||||
<TableCell
|
||||
className={`px-4 py-3 text-right tabular-nums ${pull.branch_cost?.status === "matched" ? "font-medium" : "whitespace-normal text-xs text-muted-foreground"}`}
|
||||
>
|
||||
{branchCostLabel(pull)}
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right tabular-nums">{estimateLabel(pull.estimate)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{pulls.length === 0 && (
|
||||
<TableRow>
|
||||
<TableCell className="h-32 text-center text-muted-foreground" colSpan={3}>
|
||||
{query ? `No matching ${changeName}s. Try another search.` : emptyMessage}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
{pulls.length > visibleCount && (
|
||||
<div className="flex items-center justify-between gap-4 border-t px-4 py-3">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Showing {Math.min(visibleCount, pulls.length)} of {pulls.length}
|
||||
</p>
|
||||
<Button
|
||||
variant="link"
|
||||
className="px-0"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() =>
|
||||
setPagination((current) => ({
|
||||
query,
|
||||
|
|
@ -177,26 +276,162 @@ export function ROIOverview({
|
|||
}))
|
||||
}
|
||||
>
|
||||
Load more pull requests
|
||||
Load more {changeName}s
|
||||
</Button>
|
||||
)}
|
||||
<Button variant="outline" onClick={onViewPeople}>
|
||||
Review email matches
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function ROIMetrics({ summary, branchMode }: { summary: ROISummary; branchMode: boolean }) {
|
||||
const branches = summary.branch_metrics;
|
||||
const metrics = summary.metrics;
|
||||
return (
|
||||
<dl aria-label="Spend and estimated engineering effort" className="grid grid-cols-2 @min-[760px]:grid-cols-4">
|
||||
<MetricCard
|
||||
title="Cost / estimated hour"
|
||||
value={formatMoney(branchMode ? branches?.cost_per_hour : metrics.cost_per_hour)}
|
||||
description="AI cost ÷ estimated effort"
|
||||
primary
|
||||
/>
|
||||
<MetricCard
|
||||
title="Matched AI costs"
|
||||
value={formatMoney(branchMode ? branches?.spend : metrics.matched_spend)}
|
||||
description={branchMode ? "Recorded branch requests" : "Matched gateway accounts"}
|
||||
/>
|
||||
<MetricCard
|
||||
title="Estimated effort"
|
||||
value={`${formatNumber(branchMode ? branches?.hours : metrics.output_hours)} hrs`}
|
||||
description={summary.effort_basis === "without_ai" ? "Estimated without AI" : "Check estimate assumptions"}
|
||||
/>
|
||||
<MetricCard
|
||||
title={branchMode ? "Branch coverage" : "Email coverage"}
|
||||
value={`${branchMode ? branches?.matched_pulls ?? 0 : metrics.matched_prs} / ${metrics.merged_prs}`}
|
||||
description={branchMode ? "Changes with recorded costs" : "Changes with email matches"}
|
||||
/>
|
||||
</dl>
|
||||
);
|
||||
}
|
||||
|
||||
function MetricCard({
|
||||
title,
|
||||
value,
|
||||
description,
|
||||
primary = false,
|
||||
}: {
|
||||
title: string;
|
||||
value: string;
|
||||
description: string;
|
||||
primary?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className={`min-w-0 space-y-2 p-4 sm:p-5 ${primary ? "bg-muted/30" : ""}`}>
|
||||
<dt className="min-h-8 text-xs font-medium text-muted-foreground @min-[360px]:min-h-0">{title}</dt>
|
||||
<dd className="whitespace-nowrap text-xl font-semibold tracking-tight tabular-nums @min-[600px]:text-2xl">
|
||||
{value}
|
||||
</dd>
|
||||
<dd className="text-xs leading-5 text-muted-foreground">{description}</dd>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MetricCard({ title, value }: { title: string; value: string }) {
|
||||
function ROIComparison({
|
||||
summary,
|
||||
branchMode,
|
||||
onViewPeople,
|
||||
}: {
|
||||
summary: ROISummary;
|
||||
branchMode: boolean;
|
||||
onViewPeople?: () => void;
|
||||
}) {
|
||||
const branches = summary.branch_metrics;
|
||||
const metrics = summary.metrics;
|
||||
const unavailableRate =
|
||||
metrics.output_hours > 0
|
||||
? "Spend per estimated hour is unavailable until all selected repositories can be read."
|
||||
: "Match gateway accounts to calculate costs per estimated hour.";
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader className="pb-2">
|
||||
<CardDescription>{title}</CardDescription>
|
||||
<CardTitle className="break-words text-2xl tabular-nums">{value}</CardTitle>
|
||||
</CardHeader>
|
||||
</Card>
|
||||
<div className="border-t">
|
||||
{!branchMode && metrics.cohort_people === 0 && onViewPeople && (
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b px-5 py-3">
|
||||
<p className="text-sm text-muted-foreground">Match people to gateway accounts to see their AI costs.</p>
|
||||
<Button variant="outline" size="sm" onClick={onViewPeople}>
|
||||
Match people
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
<details className="group">
|
||||
<summary className="flex cursor-pointer list-none flex-wrap items-center gap-x-3 gap-y-1 px-5 py-3 text-xs [&::-webkit-details-marker]:hidden">
|
||||
<span className="flex items-center gap-2 font-medium">
|
||||
<ChevronDown aria-hidden="true" className="size-3.5 shrink-0 transition-transform group-open:rotate-180" />
|
||||
How this is calculated
|
||||
</span>
|
||||
<span className="text-muted-foreground sm:ml-auto">
|
||||
{formatMoney(branchMode ? branches?.unlinked_spend : metrics.excluded_spend)}{" "}
|
||||
{branchMode ? "in unmatched costs" : "excluded from calculation"}
|
||||
</span>
|
||||
</summary>
|
||||
<div className="space-y-3 border-t bg-muted/20 px-5 py-4 text-sm leading-6 text-muted-foreground">
|
||||
<p>{effortNote(summary.effort_basis)}</p>
|
||||
{branchMode ? (
|
||||
<>
|
||||
<p>
|
||||
Only branches with matched request costs and complete effort estimates enter the calculation. Costs
|
||||
cover retained requests in this report’s UTC dates, not the branch’s lifetime.
|
||||
</p>
|
||||
<p>
|
||||
Open a change below to find its repository and branch tags. Send both with each gateway request. Email
|
||||
matching is not required. Shared branches stay ambiguous so their costs are not counted twice.
|
||||
</p>
|
||||
<p>
|
||||
{formatMoney(branches?.total_tagged_spend)} in tagged costs was found for these repositories. Costs
|
||||
without a unique, fully estimated change stay unmatched.
|
||||
</p>
|
||||
{(summary.unlinked_branches?.length ?? 0) > 0 && (
|
||||
<div className="space-y-2 border-t pt-3">
|
||||
<h3 className="font-medium text-foreground">Unmatched branches</h3>
|
||||
<ul className="divide-y">
|
||||
{summary.unlinked_branches?.map((row) => (
|
||||
<li
|
||||
key={`${row.repo}:${row.branch}`}
|
||||
className="flex items-start justify-between gap-4 py-2 text-xs"
|
||||
>
|
||||
<span className="min-w-0 break-all">
|
||||
{row.repo}
|
||||
<br />
|
||||
{row.branch}
|
||||
</span>
|
||||
<span className="shrink-0 tabular-nums">{formatMoney(row.spend)}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<p>
|
||||
{metrics.cost_per_hour != null
|
||||
? `${formatMoney(metrics.matched_spend)} AI costs ÷ ${formatNumber(metrics.output_hours)} estimated hours = ${formatMoney(metrics.cost_per_hour)} per estimated hour.`
|
||||
: unavailableRate}
|
||||
</p>
|
||||
<p>
|
||||
Includes {metrics.cohort_people} matched {metrics.cohort_people === 1 ? "person" : "people"} with
|
||||
complete estimates. Costs include each person’s full gateway usage across repositories during this UTC
|
||||
period.
|
||||
</p>
|
||||
{onViewPeople && (
|
||||
<Button variant="link" className="h-auto p-0" onClick={onViewPeople}>
|
||||
Review email matches
|
||||
</Button>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -221,93 +456,110 @@ export function ROIPeopleView({
|
|||
};
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex justify-end">
|
||||
<div className="@container overflow-hidden rounded-xl border">
|
||||
<ROIMetrics summary={summary} branchMode={false} />
|
||||
<ROIComparison summary={summary} branchMode={false} />
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center justify-between gap-4">
|
||||
<div className="space-y-1">
|
||||
<h2 className="text-base font-semibold">People and account matches</h2>
|
||||
<p className="text-sm text-muted-foreground">Select a person to match their gateway email.</p>
|
||||
</div>
|
||||
<Button variant="outline" onClick={exportCsv}>
|
||||
<Download />
|
||||
Export CSV
|
||||
</Button>
|
||||
</div>
|
||||
<p className="text-sm leading-relaxed text-muted-foreground">
|
||||
{effortNote(summary.effort_basis)} Spend includes each person’s full gateway usage for this period. This does
|
||||
not measure hours saved by AI or financial returns.
|
||||
</p>
|
||||
<Card>
|
||||
<CardContent className="pt-6">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Person</TableHead>
|
||||
<TableHead className="text-right">Gateway spend</TableHead>
|
||||
<TableHead className="text-right">Estimated hours</TableHead>
|
||||
<TableHead className="text-right">Spend / estimated hour</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{summary.people.map((person) => (
|
||||
<TableRow key={person.id}>
|
||||
<TableCell>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{person.logins.length ? (
|
||||
person.logins.map((login) =>
|
||||
readOnly ? (
|
||||
<span key={login}>{login}</span>
|
||||
) : (
|
||||
<Button
|
||||
key={login}
|
||||
variant="link"
|
||||
className="h-auto p-0"
|
||||
onClick={() => onMatch(person, login)}
|
||||
>
|
||||
{login}
|
||||
</Button>
|
||||
),
|
||||
)
|
||||
) : (
|
||||
<span>Unassigned gateway spend</span>
|
||||
)}
|
||||
<div className="overflow-hidden rounded-xl border">
|
||||
<Table className="min-w-[640px]">
|
||||
<TableHeader className="bg-muted/40">
|
||||
<TableRow className="hover:bg-transparent">
|
||||
<TableHead className="px-4 text-xs text-muted-foreground">Person</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs text-muted-foreground">AI cost</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs text-muted-foreground">Estimated effort</TableHead>
|
||||
<TableHead className="px-4 text-right text-xs text-muted-foreground">Cost / est. hour</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{summary.people.map((person) => (
|
||||
<TableRow key={person.id}>
|
||||
<TableCell className="px-4 py-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{person.logins.length ? (
|
||||
person.logins.map((login) =>
|
||||
readOnly ? (
|
||||
<span key={login}>{login}</span>
|
||||
) : (
|
||||
<Button
|
||||
key={login}
|
||||
variant="link"
|
||||
className="h-auto p-0"
|
||||
onClick={() => onMatch(person, login)}
|
||||
>
|
||||
{login}
|
||||
</Button>
|
||||
),
|
||||
)
|
||||
) : (
|
||||
<span>Unassigned gateway spend</span>
|
||||
)}
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{person.match_methods.some(
|
||||
(method) =>
|
||||
["manual", "commit email", "profile email"].includes(method) && person.spend != null,
|
||||
) ? (
|
||||
<span className="text-xs text-emerald-700">Matched</span>
|
||||
) : (
|
||||
<span className="text-xs text-muted-foreground">Unmatched</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">{person.email || "Email unavailable"}</p>
|
||||
{person.logins.some((login) => identityMap[login.toLowerCase()]) && (
|
||||
<p className="text-xs text-muted-foreground">Manual email match</p>
|
||||
)}
|
||||
{!person.eligible && <p className="text-xs text-muted-foreground">Excluded from ratio</p>}
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{formatMoney(person.spend)}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">
|
||||
{person.estimated_prs > 0 ? `${formatNumber(person.hours)} hrs` : "—"}
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{person.prs} {person.prs === 1 ? "PR" : "PRs"}
|
||||
{person.pending_prs > 0 ? ` · ${person.pending_prs} pending` : ""}
|
||||
</p>
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{formatMoney(person.cost_per_hour)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{summary.people.length === 0 && (
|
||||
<TableRow>
|
||||
<TableCell className="text-center text-muted-foreground" colSpan={4}>
|
||||
No people in this period.
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<details className="rounded-lg border p-4 text-sm">
|
||||
<summary className="cursor-pointer font-medium">How email matching works</summary>
|
||||
<p className="mt-3 text-muted-foreground">
|
||||
Matches use the author’s public GitHub email or commit emails associated with their GitHub account. Email
|
||||
matching ignores case. Private, noreply, and ambiguous emails stay unmatched. Manual matches take priority.
|
||||
People with no spend record or incomplete PR estimates are excluded from the ratio.
|
||||
</p>
|
||||
)
|
||||
? "Matched"
|
||||
: "Unmatched"}
|
||||
</span>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-muted-foreground">
|
||||
{person.email || "No public email"}
|
||||
{person.logins.some((login) => identityMap[login.toLowerCase()]) ? " · Manual match" : ""}
|
||||
</p>
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right tabular-nums">{formatMoney(person.spend)}</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right tabular-nums">
|
||||
{person.estimated_prs > 0 ? `${formatNumber(person.hours)} hrs` : "—"}
|
||||
<p className="mt-1 text-xs text-muted-foreground">
|
||||
{person.prs} {person.prs === 1 ? "change" : "changes"}
|
||||
{person.pending_prs > 0 ? ` · ${person.pending_prs} pending` : ""}
|
||||
</p>
|
||||
</TableCell>
|
||||
<TableCell className="px-4 py-3 text-right tabular-nums">
|
||||
{formatMoney(person.cost_per_hour)}
|
||||
{!person.eligible && <p className="mt-1 text-xs text-muted-foreground">Not included</p>}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{summary.people.length === 0 && (
|
||||
<TableRow>
|
||||
<TableCell className="h-32 text-center text-muted-foreground" colSpan={4}>
|
||||
No people in this period.
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
<details className="group rounded-xl border">
|
||||
<summary className="flex cursor-pointer list-none items-center gap-2 px-5 py-3 text-xs font-medium [&::-webkit-details-marker]:hidden">
|
||||
<ChevronDown aria-hidden="true" className="size-3.5 transition-transform group-open:rotate-180" />
|
||||
How email matching works
|
||||
</summary>
|
||||
<div className="space-y-3 border-t px-5 py-4 text-sm leading-6 text-muted-foreground">
|
||||
<p>
|
||||
Matches use the author’s public profile email
|
||||
{summary.source_provider === "gitlab"
|
||||
? "."
|
||||
: " or commit emails associated with their GitHub account."}{" "}
|
||||
Private, noreply, and ambiguous emails stay unmatched. Manual matches take priority.
|
||||
</p>
|
||||
<p>
|
||||
Costs include each person’s full gateway usage for this period. People without a spend record or with
|
||||
incomplete estimates are not included in the calculation.
|
||||
</p>
|
||||
<p>{effortNote(summary.effort_basis)}</p>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -38,9 +38,13 @@ export default function ROISettingsPanel({
|
|||
readOnly: boolean;
|
||||
syncDisabled: boolean;
|
||||
}) {
|
||||
const initialStep = initialSettings.has_github_token ? 1 : 0;
|
||||
const [provider, setProvider] = React.useState<"github" | "gitlab">(initialSettings.source_provider ?? "github");
|
||||
const sourceName = provider === "gitlab" ? "GitLab" : "GitHub";
|
||||
const savedToken = provider === "gitlab" ? initialSettings.has_gitlab_token : initialSettings.has_github_token;
|
||||
const savedUrl = provider === "gitlab" ? initialSettings.gitlab_api_url : initialSettings.github_api_url;
|
||||
const initialStep = savedToken ? 1 : 0;
|
||||
const [step, setStep] = React.useState(initialSettings.ready ? 2 : initialStep);
|
||||
const [apiUrl, setApiUrl] = React.useState(initialSettings.github_api_url);
|
||||
const [apiUrl, setApiUrl] = React.useState(savedUrl ?? "https://gitlab.com/api/v4");
|
||||
const [token, setToken] = React.useState("");
|
||||
const [clearToken, setClearToken] = React.useState(false);
|
||||
const [repos, setRepos] = React.useState(initialSettings.repos);
|
||||
|
|
@ -62,8 +66,27 @@ export default function ROISettingsPanel({
|
|||
const [error, setError] = React.useState<string | null>(null);
|
||||
const [message, setMessage] = React.useState<string | null>(null);
|
||||
|
||||
const canLoadRepositories =
|
||||
initialSettings.has_github_token && !token.trim() && apiUrl === initialSettings.github_api_url;
|
||||
const sourceUnchanged = provider === (initialSettings.source_provider ?? "github") && apiUrl === savedUrl;
|
||||
const credentialsSaved = sourceUnchanged && !token.trim() && !clearToken;
|
||||
const canLoadRepositories = credentialsSaved && (provider === "gitlab" || savedToken);
|
||||
const tokenHelp =
|
||||
provider === "gitlab"
|
||||
? "For private projects, use a token with read_api scope and project access."
|
||||
: "For private repositories, use a token with read access to contents and pull requests.";
|
||||
|
||||
const changeProvider = (next: "github" | "gitlab") => {
|
||||
setProvider(next);
|
||||
setApiUrl(
|
||||
next === "gitlab"
|
||||
? initialSettings.gitlab_api_url ?? "https://gitlab.com/api/v4"
|
||||
: initialSettings.github_api_url,
|
||||
);
|
||||
setToken("");
|
||||
setClearToken(false);
|
||||
setRepos([]);
|
||||
setAvailableRepos([]);
|
||||
setHasMoreRepos(false);
|
||||
};
|
||||
|
||||
const loadRepositories = async (page: number) => {
|
||||
if (!accessToken || !canLoadRepositories) return;
|
||||
|
|
@ -86,8 +109,10 @@ export default function ROISettingsPanel({
|
|||
|
||||
const saveSettings = async () => {
|
||||
if (!accessToken || readOnly) return false;
|
||||
const tokenValue = clearToken ? null : token.trim() || undefined;
|
||||
const body: ROISettingsUpdate = {
|
||||
github_api_url: apiUrl,
|
||||
source_provider: provider,
|
||||
...(provider === "gitlab" ? { gitlab_api_url: apiUrl } : { github_api_url: apiUrl }),
|
||||
repos,
|
||||
estimator_model: model,
|
||||
estimator_prompt: prompt,
|
||||
|
|
@ -95,8 +120,7 @@ export default function ROISettingsPanel({
|
|||
update_interval_minutes: Number(intervalHours) * 60,
|
||||
...(clearEstimatorKey ? { estimator_key: null } : {}),
|
||||
...(estimatorKey.trim() ? { estimator_key: estimatorKey.trim() } : {}),
|
||||
...(clearToken ? { github_token: null } : {}),
|
||||
...(token.trim() ? { github_token: token.trim() } : {}),
|
||||
...(provider === "gitlab" ? { gitlab_token: tokenValue } : { github_token: tokenValue }),
|
||||
};
|
||||
try {
|
||||
setBusy(true);
|
||||
|
|
@ -122,6 +146,8 @@ export default function ROISettingsPanel({
|
|||
event.preventDefault();
|
||||
if (!(await saveSettings())) return;
|
||||
if (onboarding && step === 0) {
|
||||
setStep(1);
|
||||
if (!(token.trim() || savedToken) || clearToken) return;
|
||||
try {
|
||||
const result = await apiClient.get<ROIRepositoriesResponse>("/roi-calculator/repositories", { accessToken });
|
||||
setAvailableRepos(result.repositories);
|
||||
|
|
@ -157,7 +183,7 @@ export default function ROISettingsPanel({
|
|||
try {
|
||||
const updated = await apiClient.post<ROISettings>("/roi-calculator/setup/reset", { accessToken });
|
||||
setRepos([]);
|
||||
setStep(updated.has_github_token ? 1 : 0);
|
||||
setStep(0);
|
||||
setResetOpen(false);
|
||||
onReset(updated);
|
||||
} catch (reason) {
|
||||
|
|
@ -173,27 +199,25 @@ export default function ROISettingsPanel({
|
|||
|
||||
const formDisabled = busy || syncDisabled;
|
||||
const runDisabled = formDisabled || !repos.length || !model;
|
||||
const githubUrlChanged = apiUrl !== initialSettings.github_api_url;
|
||||
const missingReplacementToken = initialSettings.has_github_token && githubUrlChanged && !token.trim();
|
||||
const stepReady = [Boolean(token.trim() || initialSettings.has_github_token), repos.length > 0, Boolean(model)][step];
|
||||
const sourceUrlChanged = apiUrl !== savedUrl;
|
||||
const missingReplacementToken = savedToken && sourceUrlChanged && !token.trim();
|
||||
const stepReady = [true, repos.length > 0, Boolean(model)][step];
|
||||
const onboardingLabel = step < 2 ? "Continue" : "Start backfill";
|
||||
const submitLabel = onboarding ? onboardingLabel : "Save settings";
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<h2 className="text-base leading-normal font-medium">
|
||||
{onboarding
|
||||
? ["Connect GitHub to get started", "Choose repositories", "Choose an estimator"][step]
|
||||
: "ROI Calculator settings"}
|
||||
</h2>
|
||||
<CardDescription>
|
||||
{onboarding
|
||||
? "Your gateway is already connected. Set up GitHub and an estimator to see your first report."
|
||||
: "Choose GitHub repositories and the router model used for metadata-only estimates."}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-5">
|
||||
<Card className={onboarding ? "max-w-2xl" : "border-0 py-0 shadow-none ring-0"}>
|
||||
{onboarding && (
|
||||
<CardHeader>
|
||||
<h2 className="text-base leading-normal font-medium">
|
||||
{["Connect your repositories", "Choose repositories", "Choose an estimator"][step]}
|
||||
</h2>
|
||||
<CardDescription>
|
||||
Your gateway is already connected. Choose a source and an estimator for your first report.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
)}
|
||||
<CardContent className={onboarding ? "space-y-5" : "space-y-5 px-0"}>
|
||||
{error && (
|
||||
<p className="text-sm text-destructive" role="alert">
|
||||
{error}
|
||||
|
|
@ -205,16 +229,34 @@ export default function ROISettingsPanel({
|
|||
</p>
|
||||
)}
|
||||
{onboarding && (
|
||||
<p className="text-sm text-muted-foreground">Step {step + 1} of 3 · GitHub / Repositories / Estimator</p>
|
||||
<p className="text-sm text-muted-foreground">Step {step + 1} of 3 · Source / Repositories / Estimator</p>
|
||||
)}
|
||||
<form className="space-y-5" onSubmit={(event) => void submit(event)}>
|
||||
<fieldset disabled={busy || syncDisabled || readOnly} className="space-y-5">
|
||||
{(!onboarding || step === 0) && (
|
||||
<>
|
||||
<section className="space-y-4">
|
||||
{!onboarding && <h3 className="font-semibold">Connection</h3>}
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="roi-source">Repository source</Label>
|
||||
<select
|
||||
id="roi-source"
|
||||
className="h-9 min-w-0 rounded-md border bg-background pl-3 pr-9 text-sm"
|
||||
value={provider}
|
||||
onChange={(event) => changeProvider(event.target.value as "github" | "gitlab")}
|
||||
>
|
||||
<option value="github">GitHub</option>
|
||||
<option value="gitlab">GitLab</option>
|
||||
</select>
|
||||
{provider !== (initialSettings.source_provider ?? "github") && !onboarding && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Switching source starts a new report and resets email matches.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<details>
|
||||
<summary className="cursor-pointer text-sm text-muted-foreground">GitHub Enterprise settings</summary>
|
||||
<summary className="cursor-pointer text-sm text-muted-foreground">Self-hosted {sourceName}</summary>
|
||||
<div className="mt-3 grid gap-2">
|
||||
<Label htmlFor="roi-github-url">GitHub API URL</Label>
|
||||
<Label htmlFor="roi-github-url">{sourceName} API URL</Label>
|
||||
<Input
|
||||
disabled={readOnly}
|
||||
id="roi-github-url"
|
||||
|
|
@ -224,7 +266,7 @@ export default function ROISettingsPanel({
|
|||
</div>
|
||||
</details>
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="roi-github-token">GitHub token</Label>
|
||||
<Label htmlFor="roi-github-token">{sourceName} token (optional for public repositories)</Label>
|
||||
<Input
|
||||
autoComplete="new-password"
|
||||
disabled={readOnly}
|
||||
|
|
@ -235,22 +277,20 @@ export default function ROISettingsPanel({
|
|||
setToken(event.target.value);
|
||||
setClearToken(false);
|
||||
}}
|
||||
placeholder={initialSettings.has_github_token ? "Token saved" : "Enter a GitHub token"}
|
||||
placeholder={savedToken ? "Token saved" : `Enter a ${sourceName} token`}
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{initialSettings.has_github_token
|
||||
? "A token is saved securely and is never shown here."
|
||||
: "Save a token to list repositories and read private repository metadata."}
|
||||
{savedToken ? "A token is saved securely and is never shown here." : tokenHelp}
|
||||
</p>
|
||||
{missingReplacementToken && (
|
||||
<p className="text-xs text-amber-700">
|
||||
Changing the GitHub API URL clears the saved token. Enter a replacement token to keep access.
|
||||
Changing the API URL clears the saved token. Enter a replacement token to keep access.
|
||||
</p>
|
||||
)}
|
||||
{initialSettings.has_github_token && (
|
||||
{savedToken && (
|
||||
<label className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
aria-label="Clear saved GitHub token"
|
||||
aria-label={`Clear saved ${sourceName} token`}
|
||||
checked={clearToken}
|
||||
disabled={readOnly}
|
||||
type="checkbox"
|
||||
|
|
@ -260,11 +300,14 @@ export default function ROISettingsPanel({
|
|||
</label>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
</section>
|
||||
)}
|
||||
{(!onboarding || step === 1) && (
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="roi-repository-search">Repositories</Label>
|
||||
<section className={onboarding ? "space-y-4" : "space-y-4 border-t pt-5"}>
|
||||
<h3 className="font-semibold">Repositories</h3>
|
||||
<Label className="sr-only" htmlFor="roi-repository-search">
|
||||
Search repositories
|
||||
</Label>
|
||||
<div className="flex gap-2">
|
||||
<Input
|
||||
id="roi-repository-search"
|
||||
|
|
@ -283,7 +326,9 @@ export default function ROISettingsPanel({
|
|||
</div>
|
||||
{!canLoadRepositories && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Save the GitHub token and API URL before loading repositories.
|
||||
{provider === "github" && !savedToken
|
||||
? "Save a GitHub token to browse repositories, or add a public repository by name."
|
||||
: "Save the source and connection settings before loading repositories."}
|
||||
</p>
|
||||
)}
|
||||
{repos.length > 0 && (
|
||||
|
|
@ -302,12 +347,13 @@ export default function ROISettingsPanel({
|
|||
))}
|
||||
</div>
|
||||
)}
|
||||
<details>
|
||||
<summary className="cursor-pointer text-xs text-muted-foreground">Add a repository by name</summary>
|
||||
<div>
|
||||
<Label htmlFor="roi-repository-name">Add a repository by name</Label>
|
||||
<div className="mt-2 flex gap-2">
|
||||
<Input
|
||||
id="roi-repository-name"
|
||||
aria-label="Repository name"
|
||||
placeholder="owner/repository"
|
||||
placeholder={provider === "gitlab" ? "group/subgroup/project" : "owner/repository"}
|
||||
value={repositoryName}
|
||||
onChange={(e) => setRepositoryName(e.target.value)}
|
||||
/>
|
||||
|
|
@ -323,30 +369,27 @@ export default function ROISettingsPanel({
|
|||
Add
|
||||
</Button>
|
||||
</div>
|
||||
</details>
|
||||
<div className="max-h-56 space-y-2 overflow-y-auto rounded-md border p-3">
|
||||
{availableRepos.map((repository) => (
|
||||
<label key={repository.name} className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
aria-label={`Select ${repository.name}`}
|
||||
checked={repos.includes(repository.name)}
|
||||
disabled={readOnly}
|
||||
type="checkbox"
|
||||
onChange={() => toggleRepository(repository.name)}
|
||||
/>
|
||||
<span>{repository.name}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{repository.visibility}
|
||||
{repository.archived ? " · archived" : ""}
|
||||
</span>
|
||||
</label>
|
||||
))}
|
||||
{availableRepos.length === 0 && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Load repositories to choose which pull requests to analyze.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{availableRepos.length > 0 && (
|
||||
<div className="max-h-56 space-y-2 overflow-y-auto rounded-md border p-3">
|
||||
{availableRepos.map((repository) => (
|
||||
<label key={repository.name} className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
aria-label={`Select ${repository.name}`}
|
||||
checked={repos.includes(repository.name)}
|
||||
disabled={readOnly}
|
||||
type="checkbox"
|
||||
onChange={() => toggleRepository(repository.name)}
|
||||
/>
|
||||
<span>{repository.name}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{repository.visibility}
|
||||
{repository.archived ? " · archived" : ""}
|
||||
</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{hasMoreRepos && (
|
||||
<Button
|
||||
type="button"
|
||||
|
|
@ -358,15 +401,16 @@ export default function ROISettingsPanel({
|
|||
Load more repositories
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
{(!onboarding || step === 2) && (
|
||||
<>
|
||||
<section className={onboarding ? "space-y-4" : "space-y-4 border-t pt-5"}>
|
||||
{!onboarding && <h3 className="font-semibold">Estimation and updates</h3>}
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="roi-estimator-model">Estimator model</Label>
|
||||
<select
|
||||
id="roi-estimator-model"
|
||||
className="h-9 rounded-md border bg-background px-3 text-sm"
|
||||
className="h-9 min-w-0 rounded-md border bg-background pl-3 pr-9 text-sm"
|
||||
disabled={readOnly}
|
||||
value={model}
|
||||
onChange={(event) => setModel(event.target.value)}
|
||||
|
|
@ -395,33 +439,35 @@ export default function ROISettingsPanel({
|
|||
/>
|
||||
</div>
|
||||
</details>
|
||||
<div className="grid max-w-xs gap-2">
|
||||
<Label htmlFor="roi-backfill-days">Backfill days</Label>
|
||||
<Input
|
||||
id="roi-backfill-days"
|
||||
min={1}
|
||||
max={3650}
|
||||
type="number"
|
||||
disabled={readOnly}
|
||||
value={backfillDays}
|
||||
onChange={(event) => setBackfillDays(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<div className="grid max-w-xs gap-2">
|
||||
<Label htmlFor="roi-interval">Update interval (hours)</Label>
|
||||
<Input
|
||||
id="roi-interval"
|
||||
type="number"
|
||||
min={0}
|
||||
max={720}
|
||||
step="any"
|
||||
required
|
||||
value={intervalHours}
|
||||
onChange={(e) => setIntervalHours(e.target.value)}
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
0 for manual updates; otherwise at least 5 minutes. Updates run while the gateway is running.
|
||||
</p>
|
||||
<div className="grid gap-4 sm:grid-cols-2">
|
||||
<div className="grid content-start gap-2">
|
||||
<Label htmlFor="roi-backfill-days">Backfill days</Label>
|
||||
<Input
|
||||
id="roi-backfill-days"
|
||||
min={1}
|
||||
max={3650}
|
||||
type="number"
|
||||
disabled={readOnly}
|
||||
value={backfillDays}
|
||||
onChange={(event) => setBackfillDays(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="roi-interval">Update interval (hours)</Label>
|
||||
<Input
|
||||
id="roi-interval"
|
||||
type="number"
|
||||
min={0}
|
||||
max={720}
|
||||
step="any"
|
||||
required
|
||||
value={intervalHours}
|
||||
onChange={(e) => setIntervalHours(e.target.value)}
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Use 0 for manual updates. Automatic updates require at least 5 minutes and a running gateway.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<details>
|
||||
<summary className="cursor-pointer text-sm text-muted-foreground">Advanced settings</summary>
|
||||
|
|
@ -463,13 +509,13 @@ export default function ROISettingsPanel({
|
|||
</div>
|
||||
</details>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Estimates use pull request metadata, without source code. Hours represent estimated effort without AI,
|
||||
not measured hours saved.
|
||||
Estimates use descriptions, file counts, and commit messages, without source code. Hours represent
|
||||
estimated effort without AI, not measured hours saved.
|
||||
</p>
|
||||
</>
|
||||
</section>
|
||||
)}
|
||||
{!readOnly && (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<div className="flex flex-wrap gap-2 border-t pt-5">
|
||||
{onboarding && step > 0 && (
|
||||
<Button
|
||||
type="button"
|
||||
|
|
|
|||
|
|
@ -8,11 +8,14 @@ import {
|
|||
formatMoney,
|
||||
formatNumber,
|
||||
formatSyncedAt,
|
||||
highestCostPulls,
|
||||
peopleCsv,
|
||||
} from "./roiCalculatorData";
|
||||
import type { ROIPull } from "./roiCalculatorData";
|
||||
|
||||
const pull = (overrides: Partial<ROIPull>): ROIPull => ({
|
||||
source_repo: "github.com/org/repo",
|
||||
source_branch: "feature",
|
||||
repo: "org/repo",
|
||||
number: 42,
|
||||
title: "Improve request routing",
|
||||
|
|
@ -39,9 +42,20 @@ const summary = {
|
|||
};
|
||||
|
||||
describe("ROI calculator display helpers", () => {
|
||||
it("ranks only attributed PR costs, limits the overview to five and preserves report order", () => {
|
||||
const pulls = [2, 6, 1, 4, 3, 5].map((spend) =>
|
||||
pull({ number: spend, branch_cost: { status: "matched", spend, requests: 1, cost_per_hour: spend } }),
|
||||
);
|
||||
pulls.push(pull({ number: 99, branch_cost: { status: "ambiguous", spend: 99, requests: 1, cost_per_hour: null } }));
|
||||
pulls.push(pull({ number: 100 }));
|
||||
expect(highestCostPulls(pulls).map((item) => item.number)).toEqual([6, 5, 4, 3, 2]);
|
||||
expect(pulls.map((item) => item.number)).toEqual([2, 6, 1, 4, 3, 5, 99, 100]);
|
||||
});
|
||||
it("formats spend and estimated hours without losing null values", () => {
|
||||
expect(formatMoney(1234.5)).toBe("$1,234.50");
|
||||
expect(formatMoney(0.0001)).toBe("<$0.01");
|
||||
expect(formatMoney(0.0001)).toBe("$0.0001");
|
||||
expect(formatMoney(0.000186)).toBe("$0.000186");
|
||||
expect(formatMoney(0.0000001)).toBe("<$0.000001");
|
||||
expect(formatMoney(0)).toBe("$0.00");
|
||||
expect(formatMoney(null)).toBe("—");
|
||||
expect(formatNumber(4.25)).toBe("4.3");
|
||||
|
|
@ -56,13 +70,14 @@ describe("ROI calculator display helpers", () => {
|
|||
it("keeps caveat copy tied to the estimate basis and reports match coverage", () => {
|
||||
expect(effortNote("without_ai")).toContain("not actual hours worked or hours saved");
|
||||
expect(effortNote(null)).toContain("Earlier estimates");
|
||||
expect(coverageLabel(summary)).toBe("1 of 2 PRs have email matches");
|
||||
expect(coverageLabel(summary)).toBe("1 of 2 matched");
|
||||
});
|
||||
|
||||
it("labels estimates and filters PRs by title, repository, number, or login", () => {
|
||||
const matchingPull = pull({});
|
||||
expect(estimateLabel(matchingPull.estimate)).toBe("4.5 hrs");
|
||||
expect(estimateLabel({ status: "needs_review", hours: null, reasoning: "", cached: false })).toBe("Needs review");
|
||||
const incompleteEstimate = { status: "needs_review" as const, hours: null, reasoning: "", cached: false };
|
||||
expect(estimateLabel(incompleteEstimate)).toBe("Needs review");
|
||||
expect(filterPulls([matchingPull], "ROUTING")).toEqual([matchingPull]);
|
||||
expect(filterPulls([matchingPull], "nobody")).toEqual([]);
|
||||
});
|
||||
|
|
@ -75,6 +90,9 @@ it("exports precise spend, cohort eligibility and safely quoted CSV values", ()
|
|||
effort_basis: "without_ai",
|
||||
people: [
|
||||
{
|
||||
id: "export-person",
|
||||
estimated_prs: 1,
|
||||
match_methods: ["manual"],
|
||||
email: '=HYPERLINK("bad")',
|
||||
logins: ["alice", "bob"],
|
||||
spend: 0.0001,
|
||||
|
|
|
|||
|
|
@ -25,8 +25,14 @@ const SYNCED_AT_FORMAT_OPTIONS: Intl.DateTimeFormatOptions = {
|
|||
|
||||
export const formatMoney = (value: number | null | undefined): string => {
|
||||
if (value == null) return "—";
|
||||
if (value > 0 && value < 0.01) return "<$0.01";
|
||||
return new Intl.NumberFormat("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 2 }).format(value);
|
||||
if (value > 0 && value < 0.000001) return "<$0.000001";
|
||||
const options: Intl.NumberFormatOptions = {
|
||||
style: "currency",
|
||||
currency: "USD",
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: Math.abs(value) > 0 && Math.abs(value) < 0.01 ? 6 : 2,
|
||||
};
|
||||
return new Intl.NumberFormat("en-US", options).format(value);
|
||||
};
|
||||
|
||||
export const formatNumber = (value: number | null | undefined): string =>
|
||||
|
|
@ -45,7 +51,8 @@ export const effortNote = (basis: string | null | undefined): string =>
|
|||
|
||||
export const coverageLabel = (summary: {
|
||||
metrics: Pick<ROISummary["metrics"], "matched_prs" | "merged_prs">;
|
||||
}): string => `${summary.metrics.matched_prs} of ${summary.metrics.merged_prs} PRs have email matches`;
|
||||
source_provider?: string;
|
||||
}): string => `${summary.metrics.matched_prs} of ${summary.metrics.merged_prs} matched`;
|
||||
|
||||
export const estimateLabel = (estimate: ROIEstimate): string => {
|
||||
if (estimate.status === "estimated") return `${formatNumber(estimate.hours)} hrs`;
|
||||
|
|
@ -57,10 +64,18 @@ export const filterPulls = (pulls: ROIPull[], query: string): ROIPull[] => {
|
|||
const normalized = query.trim().toLocaleLowerCase();
|
||||
if (!normalized) return pulls;
|
||||
return pulls.filter((pull) =>
|
||||
`${pull.title} ${pull.repo} ${pull.number} ${pull.login}`.toLocaleLowerCase().includes(normalized),
|
||||
`${pull.title} ${pull.repo} ${pull.number} ${pull.login} ${pull.source_branch ?? ""}`
|
||||
.toLocaleLowerCase()
|
||||
.includes(normalized),
|
||||
);
|
||||
};
|
||||
|
||||
export const highestCostPulls = (pulls: ROIPull[]): ROIPull[] =>
|
||||
pulls
|
||||
.filter((pull) => pull.branch_cost?.status === "matched")
|
||||
.sort((left, right) => (right.branch_cost?.spend ?? 0) - (left.branch_cost?.spend ?? 0))
|
||||
.slice(0, 5);
|
||||
|
||||
export const peopleCsv = (summary: Pick<ROISummary, "people" | "start" | "end" | "effort_basis">): string => {
|
||||
const escape = (value: unknown): string => {
|
||||
const text = value == null ? "" : String(value);
|
||||
|
|
@ -83,7 +98,7 @@ export const peopleCsv = (summary: Pick<ROISummary, "people" | "start" | "end" |
|
|||
return [
|
||||
[
|
||||
"email",
|
||||
"github_logins",
|
||||
"source_logins",
|
||||
"gateway_spend_usd",
|
||||
"estimated_hours",
|
||||
"merged_prs",
|
||||
|
|
@ -99,3 +114,10 @@ export const peopleCsv = (summary: Pick<ROISummary, "people" | "start" | "end" |
|
|||
.map((row) => row.map(escape).join(","))
|
||||
.join("\r\n");
|
||||
};
|
||||
|
||||
export const branchCostLabel = (pull: ROIPull): string => {
|
||||
if (!pull.branch_cost || pull.branch_cost.status === "unavailable") return "Sync to calculate";
|
||||
if (pull.branch_cost.status === "ambiguous") return "Ambiguous branch";
|
||||
if (pull.branch_cost.status === "unattributed") return "No tagged requests";
|
||||
return formatMoney(pull.branch_cost.spend);
|
||||
};
|
||||
|
|
|
|||
106
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -41250,6 +41250,67 @@ export interface components {
|
|||
};
|
||||
} | null;
|
||||
};
|
||||
/** ROIBranchAttribution */
|
||||
ROIBranchAttribution: {
|
||||
/** Branch */
|
||||
branch: string;
|
||||
/** Repo */
|
||||
repo: string;
|
||||
/**
|
||||
* Requests
|
||||
* @default 0
|
||||
*/
|
||||
requests: number;
|
||||
/** Spend */
|
||||
spend?: number | null;
|
||||
/**
|
||||
* Status
|
||||
* @default unattributed
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "matched" | "unattributed" | "ambiguous" | "unavailable";
|
||||
};
|
||||
/** ROIBranchMetrics */
|
||||
ROIBranchMetrics: {
|
||||
/** Cost Per Hour */
|
||||
cost_per_hour?: number | null;
|
||||
/**
|
||||
* Hours
|
||||
* @default 0
|
||||
*/
|
||||
hours: number;
|
||||
/**
|
||||
* Matched Pulls
|
||||
* @default 0
|
||||
*/
|
||||
matched_pulls: number;
|
||||
/**
|
||||
* Spend
|
||||
* @default 0
|
||||
*/
|
||||
spend: number;
|
||||
/**
|
||||
* Total Tagged Spend
|
||||
* @default 0
|
||||
*/
|
||||
total_tagged_spend: number;
|
||||
/**
|
||||
* Unlinked Spend
|
||||
* @default 0
|
||||
*/
|
||||
unlinked_spend: number;
|
||||
};
|
||||
/** ROIBranchSpend */
|
||||
ROIBranchSpend: {
|
||||
/** Branch */
|
||||
branch: string;
|
||||
/** Repo */
|
||||
repo: string;
|
||||
/** Requests */
|
||||
requests: number;
|
||||
/** Spend */
|
||||
spend: number;
|
||||
};
|
||||
/** ROIEstimateResponse */
|
||||
ROIEstimateResponse: {
|
||||
/**
|
||||
|
|
@ -41346,6 +41407,7 @@ export interface components {
|
|||
ROIPullResponse: {
|
||||
/** Additions */
|
||||
additions: number;
|
||||
branch_cost?: components["schemas"]["ROIBranchAttribution"];
|
||||
/** Cache Key */
|
||||
cache_key?: string | null;
|
||||
/** Changed Files */
|
||||
|
|
@ -41377,6 +41439,16 @@ export interface components {
|
|||
profile_email: string;
|
||||
/** Repo */
|
||||
repo: string;
|
||||
/**
|
||||
* Source Branch
|
||||
* @default
|
||||
*/
|
||||
source_branch: string;
|
||||
/**
|
||||
* Source Repo
|
||||
* @default
|
||||
*/
|
||||
source_repo: string;
|
||||
/** Title */
|
||||
title: string;
|
||||
/** Url */
|
||||
|
|
@ -41418,10 +41490,20 @@ export interface components {
|
|||
estimator_prompt: string;
|
||||
/** Github Api Url */
|
||||
github_api_url: string;
|
||||
/**
|
||||
* Gitlab Api Url
|
||||
* @default https://gitlab.com/api/v4
|
||||
*/
|
||||
gitlab_api_url: string;
|
||||
/** Has Estimator Key */
|
||||
has_estimator_key: boolean;
|
||||
/** Has Github Token */
|
||||
has_github_token: boolean;
|
||||
/**
|
||||
* Has Gitlab Token
|
||||
* @default false
|
||||
*/
|
||||
has_gitlab_token: boolean;
|
||||
/** Identity Map */
|
||||
identity_map: {
|
||||
[key: string]: string;
|
||||
|
|
@ -41430,6 +41512,12 @@ export interface components {
|
|||
ready: boolean;
|
||||
/** Repos */
|
||||
repos: string[];
|
||||
/**
|
||||
* Source Provider
|
||||
* @default github
|
||||
* @enum {string}
|
||||
*/
|
||||
source_provider: "github" | "gitlab";
|
||||
/** Update Interval Minutes */
|
||||
update_interval_minutes: number;
|
||||
};
|
||||
|
|
@ -41447,13 +41535,20 @@ export interface components {
|
|||
github_api_url?: string | null;
|
||||
/** Github Token */
|
||||
github_token?: string | null;
|
||||
/** Gitlab Api Url */
|
||||
gitlab_api_url?: string | null;
|
||||
/** Gitlab Token */
|
||||
gitlab_token?: string | null;
|
||||
/** Repos */
|
||||
repos?: string[] | null;
|
||||
/** Source Provider */
|
||||
source_provider?: ("github" | "gitlab") | null;
|
||||
/** Update Interval Minutes */
|
||||
update_interval_minutes?: number | null;
|
||||
};
|
||||
/** ROISummaryResponse */
|
||||
ROISummaryResponse: {
|
||||
branch_metrics?: components["schemas"]["ROIBranchMetrics"];
|
||||
/** Effort Basis */
|
||||
effort_basis: string | null;
|
||||
/** End */
|
||||
|
|
@ -41473,12 +41568,23 @@ export interface components {
|
|||
pulls: components["schemas"]["ROIPullResponse"][];
|
||||
/** Repos */
|
||||
repos: string[];
|
||||
/**
|
||||
* Source Provider
|
||||
* @default github
|
||||
* @enum {string}
|
||||
*/
|
||||
source_provider: "github" | "gitlab";
|
||||
/** Start */
|
||||
start: string;
|
||||
/** Synced At */
|
||||
synced_at: string;
|
||||
/** Trend */
|
||||
trend: components["schemas"]["ROITrendResponse"][];
|
||||
/**
|
||||
* Unlinked Branches
|
||||
* @default []
|
||||
*/
|
||||
unlinked_branches: components["schemas"]["ROIBranchSpend"][];
|
||||
/** Warnings */
|
||||
warnings: string[];
|
||||
};
|
||||
|
|
|
|||