scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(roi): measure shipping velocity, quality, and recorded spend (#44426)

Browse source 
* feat(ui): prototype observed engineering ROI dashboard

* feat(roi): replace effort estimates with measured repository metrics

* fix(roi): finish connection recovery and generated API contracts

* fix(roi): show merged changes before accounts are linked

* fix(roi): preserve selected report tab across refreshes

* fix(roi): recover app authorization and keep detail values readable

* fix(roi): reuse the shared OAuth HTTP client

* feat(roi): combine providers and compare equal reporting periods

* docs: explain ROI metrics for first-time readers

* fix(roi): preserve connections and scheduled reports during setup

* ci(roi): assign database contracts to the active Postgres shard

* fix(roi): preserve issue counts and normalized connections

* fix(ui): compact ROI dashboard header and metrics

* fix(ui): show ROI repository count with expandable list

* fix(ui): wrap ROI controls within narrow panels

* fix(roi): restore sample report preview and simplify setup
This commit is contained in:
moe-berri 2026-10-03 16:07:34 -07:00 • committed by GitHub
parent c7e60f03de
commit 1a7023366f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 9483 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
.github/scripts/assert_ci_coverage.py vendored
View file

@ -130,37 +130,24 @@ def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, fr
text: Final = _uncommented(script.read_text())
return MappingProxyType(
{
label: frozenset(
match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)
)
label: frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body))
for label, body in SELECTION_ARM_RE.findall(text)
}
)
def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
return frozenset(
token for tokens in _unit_selection_arms(repo_root).values() for token in tokens
)
return frozenset(token for tokens in _unit_selection_arms(repo_root).values() for token in tokens)
def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
return frozenset(
scalar.value
for scalar in scalars
if scalar.key == "unit-flag" and "${{" not in scalar.value
)
return frozenset(scalar.value for scalar in scalars if scalar.key == "unit-flag" and "${{" not in scalar.value)
def _shard_tokens(
scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]
) -> frozenset[str]:
def _shard_tokens(scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]) -> frozenset[str]:
wired: Final = _wired_unit_flags(scalars)
return _invoked_test_tokens(scalars) | frozenset(
token
for label, tokens in arms.items()
if label in wired
for token in tokens
token for label, tokens in arms.items() if label in wired for token in tokens
)
@ -544,17 +531,37 @@ def _integration_groups(runner: pathlib.Path) -> dict[str, tuple[str, ...]]:
return {group: tuple(folders) for group, folders in ast.literal_eval(mapping).items()}
def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
module: Final = ast.parse(runner.read_text())
literal: Final = next(
(
node.value
for node in module.body
if isinstance(node, ast.AnnAssign)
and isinstance(node.target, ast.Name)
and node.target.id == "GITHUB_FILES"
),
None,
)
if literal is None:
return frozenset()
values: Final = literal.args[0] if isinstance(literal, ast.Call) else literal
return frozenset(ast.literal_eval(values))
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
runner: Final = repo_root / "tests/integration/run.py"
if not runner.exists():
return frozenset(), ()
groups: Final = _integration_groups(runner)
github_files: Final = _integration_github_files(runner)
integration_root: Final = repo_root / "tests/integration"
paths: Final = frozenset(
str(path.relative_to(repo_root))
for folders in groups.values()
for folder in folders
for path in (integration_root / folder).rglob("test_*.py")
if str(path.relative_to(repo_root)) not in github_files
)
browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()
@ -595,10 +602,22 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
for path in (repo_root / ".github/workflows").glob("*.y*ml")
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
)
findings: Final = tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
if any(_token_covers(token, path) for token in gha_tokens)
findings: Final = (
tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
Finding(path, "GitHub-owned integration contract has no invoking workflow")
for path in sorted(github_files)
if not any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
Finding(path, "GitHub-owned integration file is missing")
for path in sorted(github_files)
if not (repo_root / path).is_file()
)
)
browser_commands: Final = tuple(
scalar.value
@ -642,7 +661,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
return frozenset(), findings + (
Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
)
return paths | browser_paths, findings + group_findings + browser_findings + exclusion_findings
return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings
def main() -> int:

20
.github/workflows/test-postgres.yml vendored
View file

@ -45,6 +45,13 @@ jobs:
fail-fast: false
matrix:
include:
- shard: roi-database
test-path: "tests/integration/database/test_roi_observed.py"
seed: none
workers: 0
timeout-minutes: 10
job-timeout-minutes: 35
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
@ -147,7 +154,7 @@ jobs:
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }}
PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || matrix.shard == 'roi-database' && '--cov=./litellm --cov-report=xml:coverage-roi-postgres.xml' || '' }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
@ -165,3 +172,14 @@ jobs:
files: coverage-lens-postgres.xml
flags: lens-postgres
fail_ci_if_error: true
- name: Upload ROI database coverage
if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'roi-database' && !cancelled()
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
use_oidc: true
version: v11.3.1
root_dir: ${{ github.workspace }}
files: coverage-roi-postgres.xml
flags: roi-postgres
fail_ci_if_error: true

1416
litellm/proxy/_lazy_openapi_snapshot.json
View file

File diff suppressed because it is too large Load diff

243
litellm/proxy/management_endpoints/roi_calculator_endpoints.py
View file

@ -15,20 +15,29 @@ from apscheduler.schedulers.asyncio import ( # pyright: ignore[reportMissingTyp
AsyncIOScheduler,
)
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError
from starlette.types import Receive, Scope, Send
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params
)
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.management_endpoints.roi_observed_endpoints import router as observed_router
from litellm.proxy.roi_calculator.analytics import normalize_email, summarize
from litellm.proxy.roi_calculator.branch_spend import BranchSpendDatabase, read_branch_spend
from litellm.proxy.roi_calculator.estimator import CompletionCaller, EstimatorModel
from litellm.proxy.roi_calculator.github import SourceError
from litellm.proxy.roi_calculator.settings import (
active_connection,
get_roi_config_repository,
load_settings,
load_stored_settings,
read_admin,
save_settings,
write_admin,
)
from litellm.proxy.roi_calculator.source import create_source
from litellm.proxy.roi_calculator.sync import (
BranchSpendReader,
@ -62,29 +71,13 @@ from litellm.types.roi_calculator import (
)
router: Final = APIRouter()
router.include_router(observed_router)
_SETTINGS_KEY: Final = "roi_calculator_settings"
_REPORT_KEY: Final = "roi_calculator_report"
_SYNC_MANAGER: Final = SyncManager()
_ROI_TAGS: Final[list[str | Enum]] = ["roi calculator"] # mutable-ok: FastAPI requires list-valued route tags
class _StoredSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
source_provider: Literal["github", "gitlab"] = "github"
gitlab_api_url: str = "https://gitlab.com/api/v4"
gitlab_token: str = ""
github_api_url: str = "https://api.github.com"
github_token: str = ""
estimator_key: str = ""
repos: tuple[str, ...] = ()
estimator_model: str = ""
estimator_prompt: str = DEFAULT_PROMPT
backfill_days: int = Field(default=7, ge=1, le=3650)
update_interval_minutes: float = Field(default=1440, ge=0, le=43200)
identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({}))
class _RouterEstimatorParams(BaseModel):
model_config = ConfigDict(extra="ignore", from_attributes=True)
@ -108,38 +101,6 @@ class _RouterEstimatorDeployment(BaseModel):
model_info: _RouterEstimatorModelInfo | None = None
async def _read_admin(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> UserAPIKeyAuth:
if user_api_key_dict.user_role not in (
LitellmUserRoles.PROXY_ADMIN,
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
):
raise HTTPException(status_code=403, detail="Only proxy admins can access the ROI Calculator.")
return user_api_key_dict
async def _write_admin(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> UserAPIKeyAuth:
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="Only proxy admins can change ROI Calculator settings.")
return user_api_key_dict
async def get_roi_config_repository(
_user: Annotated[UserAPIKeyAuth, Depends(_read_admin)],
) -> ConfigRepository:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail=CommonProxyErrors.db_not_connected_error.value,
)
return ConfigRepository(prisma_client, use_writer=True)
def get_roi_sync_manager() -> SyncManager:
return _SYNC_MANAGER
@ -222,66 +183,6 @@ def _router_estimator_choices() -> tuple[ROIEstimatorModel, ...]:
return tuple(choice for choice in choices if choice.model_name in names)
async def _load_stored_settings(repository: ConfigRepository) -> _StoredSettings:
parameter: Final = await repository.get_param(_SETTINGS_KEY)
if parameter is None:
return _StoredSettings()
try:
return _StoredSettings.model_validate(parameter.param_value)
except ValidationError:
raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None
async def _load_settings(repository: ConfigRepository) -> ROISettings:
stored: Final = await _load_stored_settings(repository)
token: Final = decrypt_value_helper(stored.github_token, _SETTINGS_KEY) if stored.github_token else ""
try:
return ROISettings(
source_provider=stored.source_provider,
gitlab_api_url=stored.gitlab_api_url,
gitlab_token=SecretStr(decrypt_value_helper(stored.gitlab_token, _SETTINGS_KEY) or "")
if stored.gitlab_token
else SecretStr(""),
github_api_url=stored.github_api_url,
github_token=SecretStr(token or ""),
estimator_key=SecretStr(decrypt_value_helper(stored.estimator_key, _SETTINGS_KEY) or "")
if stored.estimator_key
else SecretStr(""),
update_interval_minutes=stored.update_interval_minutes,
repos=stored.repos,
estimator_model=stored.estimator_model,
estimator_prompt=stored.estimator_prompt,
backfill_days=stored.backfill_days,
identity_map=stored.identity_map,
)
except ValidationError:
raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None
async def _save_settings(
repository: ConfigRepository,
settings: ROISettings,
encrypted_token: str,
encrypted_estimator_key: str,
encrypted_gitlab_token: str = "",
) -> None:
stored: Final = _StoredSettings(
source_provider=settings.source_provider,
gitlab_api_url=settings.gitlab_api_url,
gitlab_token=encrypted_gitlab_token,
github_api_url=settings.github_api_url,
github_token=encrypted_token,
estimator_key=encrypted_estimator_key,
update_interval_minutes=settings.update_interval_minutes,
repos=settings.repos,
estimator_model=settings.estimator_model,
estimator_prompt=settings.estimator_prompt,
backfill_days=settings.backfill_days,
identity_map=settings.identity_map,
)
await repository.set_param(_SETTINGS_KEY, stored.model_dump(mode="json"))
async def _load_report(repository: ConfigRepository, settings: ROISettings) -> ROIReport | None:
parameter: Final = await repository.get_param(_REPORT_KEY)
if parameter is None or parameter.param_value is None:
@ -302,6 +203,7 @@ def _public_settings(settings: ROISettings) -> ROISettingsResponse:
choices: Final = _router_estimator_choices()
models: Final = tuple(choice.model_name for choice in choices)
return ROISettingsResponse(
report_mode=settings.report_mode,
source_provider=settings.source_provider,
gitlab_api_url=settings.gitlab_api_url,
has_gitlab_token=bool(settings.gitlab_token.get_secret_value()),
@ -392,14 +294,14 @@ async def _test_estimator_access(settings: ROISettings) -> None:
raise HTTPException(status_code=409, detail="The estimator key could not connect to the gateway.") from None
def _gateway_user_reader(repository: ConfigRepository) -> GatewayUserReader:
def gateway_user_reader(repository: ConfigRepository) -> GatewayUserReader:
async def get_emails() -> frozenset[str]:
return await read_gateway_user_emails(spend_prisma_client(repository.prisma_client))
return get_emails
def _spend_reader(repository: ConfigRepository) -> SpendReader:
def spend_reader(repository: ConfigRepository) -> SpendReader:
async def get_spend(start: date, end: date) -> tuple[ROISpendRecord, ...]:
prisma_client: Final = spend_prisma_client(repository.prisma_client)
return await read_spend(prisma_client, start, end)
@ -407,7 +309,7 @@ def _spend_reader(repository: ConfigRepository) -> SpendReader:
return get_spend
def _branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> BranchSpendReader:
def branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> BranchSpendReader:
async def get_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]:
return await read_branch_spend(
cast( # cast-ok: PrismaWrapper delegates methods dynamically
@ -428,10 +330,10 @@ def _branch_spend_reader(repository: ConfigRepository, settings: ROISettings) ->
tags=_ROI_TAGS,
)
async def get_roi_calculator_settings(
_user: Annotated[UserAPIKeyAuth, Depends(_read_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ROISettingsResponse:
return _public_settings(await _load_settings(repository))
return _public_settings(await load_settings(repository))
@router.put(
@ -441,11 +343,11 @@ async def get_roi_calculator_settings(
)
async def update_roi_calculator_settings(
patch: ROISettingsUpdate,
_user: Annotated[UserAPIKeyAuth, Depends(_write_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ROISettingsResponse:
stored: Final = await _load_stored_settings(repository)
current: Final = await _load_settings(repository)
stored: Final = await load_stored_settings(repository)
current: Final = await load_settings(repository, stored)
if "github_api_url" in patch.model_fields_set and patch.github_api_url is None:
raise HTTPException(status_code=422, detail="GitHub API URL cannot be null.")
if "gitlab_api_url" in patch.model_fields_set and patch.gitlab_api_url is None:
@ -491,6 +393,17 @@ async def update_roi_calculator_settings(
)
try:
settings: Final = ROISettings(
report_mode=patch.report_mode or current.report_mode,
connection_type="token"
if source_changed or token_was_supplied or "gitlab_token" in patch.model_fields_set
else current.connection_type,
oauth_refresh_token=SecretStr("")
if source_changed or token_was_supplied or "gitlab_token" in patch.model_fields_set
else current.oauth_refresh_token,
oauth_expires_at=None
if source_changed or token_was_supplied or "gitlab_token" in patch.model_fields_set
else current.oauth_expires_at,
ignored_logins=() if source_changed else current.ignored_logins,
source_provider=provider,
gitlab_api_url=gitlab_url,
gitlab_token=SecretStr(gitlab_token),
@ -510,7 +423,15 @@ async def update_roi_calculator_settings(
)
except ValidationError as exc:
raise HTTPException(status_code=422, detail=exc.errors(include_context=False)) from None
await _save_settings(repository, settings, encrypted_token, encrypted_estimator_key, encrypted_gitlab)
await save_settings(
repository,
settings,
encrypted_token,
encrypted_estimator_key,
encrypted_gitlab,
revision=stored.revision,
replace_connection_id=active_connection(stored).id,
)
if source_changed:
await repository.set_param(_REPORT_KEY, None)
return _public_settings(settings)
@ -522,13 +443,13 @@ async def update_roi_calculator_settings(
tags=_ROI_TAGS,
)
async def get_roi_calculator_repositories(
_user: Annotated[UserAPIKeyAuth, Depends(_read_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_github_transport)],
query: Annotated[str, Query(max_length=200)] = "",
page: Annotated[int, Query(ge=1, le=1000)] = 1,
) -> ROIRepositoriesResponse:
github: Final = create_source(await _load_settings(repository), transport)
github: Final = create_source(await load_settings(repository), transport)
try:
repos, has_more = await github.repositories(query, page)
except SourceError as exc:
@ -550,12 +471,12 @@ async def get_roi_calculator_repositories(
tags=_ROI_TAGS,
)
async def get_roi_calculator_sync_status(
_user: Annotated[UserAPIKeyAuth, Depends(_read_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
manager: Annotated[SyncManager, Depends(get_roi_sync_manager)],
) -> ROISyncStatus:
status: Final = await SyncStore(repository.prisma_client).status() or manager.status
settings: Final = await _load_settings(repository)
settings: Final = await load_settings(repository)
report: Final = await _load_report(repository, settings)
next_update: Final = _next_update(settings, status, report)
return status.model_copy(update=MappingProxyType({"next_update": next_update.isoformat() if next_update else None}))
@ -568,25 +489,25 @@ async def get_roi_calculator_sync_status(
tags=_ROI_TAGS,
)
async def start_roi_calculator_sync(
_user: Annotated[UserAPIKeyAuth, Depends(_write_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
manager: Annotated[SyncManager, Depends(get_roi_sync_manager)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_github_transport)],
) -> ROISyncStatus:
settings: Final = await _load_settings(repository)
settings: Final = await load_settings(repository)
public: Final = _public_settings(settings)
if not public.ready:
raise HTTPException(status_code=409, detail="Connect a source, select repositories, and choose a router model.")
if not await manager.start(
settings,
repository,
_spend_reader(repository),
spend_reader(repository),
_completion_caller(settings),
transport,
_router_estimator_models(settings.estimator_model),
SyncStore(repository.prisma_client),
branch_spend_reader=_branch_spend_reader(repository, settings),
gateway_user_reader=_gateway_user_reader(repository),
branch_spend_reader=branch_spend_reader(repository, settings),
gateway_user_reader=gateway_user_reader(repository),
):
raise HTTPException(status_code=409, detail="A sync is already running.")
return manager.status
@ -598,7 +519,7 @@ async def start_roi_calculator_sync(
tags=_ROI_TAGS,
)
async def cancel_roi_calculator_sync(
_user: Annotated[UserAPIKeyAuth, Depends(_write_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
manager: Annotated[SyncManager, Depends(get_roi_sync_manager)],
) -> ROISyncStatus:
@ -614,7 +535,7 @@ async def cancel_roi_calculator_sync(
tags=_ROI_TAGS,
)
async def get_roi_calculator_report(
_user: Annotated[UserAPIKeyAuth, Depends(_read_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
mode: Literal["live", "demo"] = "live",
) -> ROIReportResponse:
@ -623,7 +544,7 @@ async def get_roi_calculator_report(
sample: Final = summarize(sample_report(datetime.now(timezone.utc)), MappingProxyType({}))
return ROIReportResponse(report=ROISummaryResponse.model_validate(sample))
settings: Final = await _load_settings(repository)
settings: Final = await load_settings(repository)
report: Final = await _load_report(repository, settings)
if report is None:
return ROIReportResponse(report=None)
@ -638,12 +559,12 @@ async def get_roi_calculator_report(
)
async def update_roi_calculator_identity_map(
update: ROIIdentityMapUpdate,
_user: Annotated[UserAPIKeyAuth, Depends(_write_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ROIIdentityMapResponse:
login: Final = update.github_login.strip().casefold()
current: Final = await _load_settings(repository)
current_stored: Final = await _load_stored_settings(repository)
current_stored: Final = await load_stored_settings(repository)
current: Final = await load_settings(repository, current_stored)
try:
normalize_source_login(login, current.source_provider)
except ValueError as exc:
@ -657,6 +578,11 @@ async def update_roi_calculator_identity_map(
else MappingProxyType({**current.identity_map, login: new_email})
)
settings: Final = ROISettings(
report_mode=current.report_mode,
connection_type=current.connection_type,
oauth_refresh_token=current.oauth_refresh_token,
oauth_expires_at=current.oauth_expires_at,
ignored_logins=current.ignored_logins,
source_provider=current.source_provider,
gitlab_api_url=current.gitlab_api_url,
gitlab_token=current.gitlab_token,
@ -670,8 +596,13 @@ async def update_roi_calculator_identity_map(
backfill_days=current.backfill_days,
identity_map=identity_map,
)
await _save_settings(
repository, settings, current_stored.github_token, current_stored.estimator_key, current_stored.gitlab_token
await save_settings(
repository,
settings,
current_stored.github_token,
current_stored.estimator_key,
current_stored.gitlab_token,
revision=current_stored.revision,
)
report: Final = await _load_report(repository, settings)
summary: Final = summarize(report, settings.identity_map) if report is not None else None
@ -683,7 +614,8 @@ async def update_roi_calculator_identity_map(
def _next_update(settings: ROISettings, status: ROISyncStatus, report: ROIReport | None) -> datetime | None:
if (
not report
settings.report_mode != "legacy"
or not report
or not settings.repos
or not settings.estimator_model
or not settings.update_interval_minutes
@ -710,12 +642,16 @@ def register_scheduled_sync(scheduler: AsyncIOScheduler) -> None:
async def run_scheduled_sync() -> None:
from litellm.proxy.management_endpoints.roi_observed_endpoints import run_observed_schedule
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return
repository: Final = ConfigRepository(prisma_client, use_writer=True)
settings: Final = await _load_settings(repository)
settings: Final = await load_settings(repository)
if settings.report_mode == "observed":
await run_observed_schedule()
return
if not settings.update_interval_minutes or not _public_settings(settings).ready:
return
store: Final = SyncStore(prisma_client)
@ -727,23 +663,23 @@ async def run_scheduled_sync() -> None:
await _SYNC_MANAGER.start(
settings,
repository,
_spend_reader(repository),
spend_reader(repository),
_completion_caller(settings),
estimator_models=_router_estimator_models(settings.estimator_model),
coordinator=store,
scheduled_interval=settings.update_interval_minutes,
branch_spend_reader=_branch_spend_reader(repository, settings),
gateway_user_reader=_gateway_user_reader(repository),
branch_spend_reader=branch_spend_reader(repository, settings),
gateway_user_reader=gateway_user_reader(repository),
)
@router.post("/roi-calculator/connections/test", tags=_ROI_TAGS)
async def test_roi_calculator_connections(
_user: Annotated[UserAPIKeyAuth, Depends(_write_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_github_transport)],
) -> ROISettingsResponse:
settings: Final = await _load_settings(repository)
settings: Final = await load_settings(repository)
public: Final = _public_settings(settings)
if not public.ready:
raise HTTPException(status_code=409, detail="Choose repositories and an available estimator model first.")
@ -760,7 +696,7 @@ async def test_roi_calculator_connections(
@router.post("/roi-calculator/setup/reset", tags=_ROI_TAGS)
async def reset_roi_calculator_setup(
_user: Annotated[UserAPIKeyAuth, Depends(_write_admin)],
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ROISettingsResponse:
from uuid import uuid4
@ -781,10 +717,17 @@ async def reset_roi_calculator_setup(
if not await store.acquire(owner, status):
raise HTTPException(status_code=409, detail="Cancel the running analysis before restarting setup.")
try:
current: Final = await _load_settings(repository)
stored: Final = await _load_stored_settings(repository)
stored: Final = await load_stored_settings(repository)
current: Final = await load_settings(repository, stored)
settings: Final = current.model_copy(update=MappingProxyType({"repos": ()}))
await _save_settings(repository, settings, stored.github_token, stored.estimator_key, stored.gitlab_token)
await save_settings(
repository,
settings,
stored.github_token,
stored.estimator_key,
stored.gitlab_token,
revision=stored.revision,
)
await store.clear_report()
return _public_settings(settings)
finally:

587
litellm/proxy/management_endpoints/roi_observed_endpoints.py Normal file
View file

@ -0,0 +1,587 @@
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import Annotated, Final
import httpx
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import JSONResponse, RedirectResponse
from pydantic import SecretStr, TypeAdapter, ValidationError
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.roi_calculator.analytics import normalize_email
from litellm.proxy.roi_calculator.github import SourceError
from litellm.proxy.roi_calculator.oauth import (
OAuthConfig,
Provider,
begin_authorization,
connected_settings,
consume_state,
exchange_code,
oauth_config,
save_grant,
)
from litellm.proxy.roi_calculator.observed_sync import ObservedSyncManager, Progress
from litellm.proxy.roi_calculator.observed_workspace import (
collect_workspace,
scoped_data,
source_details,
summarize_workspace,
)
from litellm.proxy.roi_calculator.settings import (
StoredConnection,
active_connection,
connection_id,
enable_observed_reporting,
get_roi_config_repository,
load_settings,
load_stored_settings,
read_admin,
save_connection_identities,
save_settings,
select_connection,
stored_connections,
write_admin,
)
from litellm.proxy.roi_calculator.source import create_source
from litellm.proxy.roi_calculator.sync import read_gateway_user_emails, spend_prisma_client
from litellm.proxy.roi_calculator.sync_store import SyncStore
from litellm.repositories.config_repository import ConfigRepository
from litellm.types.roi_calculator import (
ROIRepositoriesResponse,
ROIRepository,
ROISettings,
ROISyncStatus,
normalize_source_login,
)
from litellm.types.roi_observed import (
ObservedAccount,
ObservedApp,
ObservedApps,
ObservedAuthorization,
ObservedConnectionIdentities,
ObservedData,
ObservedIdentities,
ObservedIdentityUpdate,
ObservedReportResponse,
ObservedSettings,
ObservedSettingsUpdate,
)
router: Final = APIRouter(prefix="/roi-calculator/observed", tags=["roi calculator"])
_MANAGER: Final = ObservedSyncManager()
_REPORT_KEY: Final = "roi_observed_report"
def get_observed_manager() -> ObservedSyncManager:
return _MANAGER
def get_observed_transport() -> httpx.AsyncBaseTransport | None:
return None
def public_settings(settings: ROISettings) -> ObservedSettings:
return ObservedSettings(
id=connection_id(settings.source_provider, settings.source_api_url),
source_provider=settings.source_provider,
api_url=settings.source_api_url,
repos=settings.repos,
has_token=bool(
(
settings.gitlab_token if settings.source_provider == "gitlab" else settings.github_token
).get_secret_value()
),
update_interval_minutes=settings.update_interval_minutes,
ready=bool(settings.repos),
connection_type=settings.connection_type,
)
async def workspace_settings(repository: ConfigRepository) -> ObservedSettings:
stored: Final = await load_stored_settings(repository)
entries: Final = tuple(
[
public_settings(await load_settings(repository, select_connection(stored, entry)))
for entry in stored_connections(stored)
]
)
current: Final = public_settings(await load_settings(repository, stored))
return current.model_copy(update={"connections": entries, "ready": any(entry.ready for entry in entries)})
async def _data(repository: ConfigRepository) -> ObservedData | None:
saved: Final = await repository.get_param(_REPORT_KEY)
if saved is None or saved.param_value is None:
return None
try:
data: Final = ObservedData.model_validate(saved.param_value)
except ValidationError:
raise HTTPException(500, "The saved report is invalid. Sync again to rebuild it.") from None
if data.connections:
return data
settings: Final = ROISettings.model_validate(
{
"source_provider": data.source_provider,
"repos": data.repos,
("gitlab_api_url" if data.source_provider == "gitlab" else "github_api_url"): data.source_api_url,
}
)
return scoped_data(data, source_details(settings))
@router.get("/settings", response_model=ObservedSettings)
async def get_observed_settings(
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ObservedSettings:
return await workspace_settings(repository)
@router.put("/settings", response_model=ObservedSettings)
async def save_observed_settings(
patch: ObservedSettingsUpdate,
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)],
) -> ObservedSettings:
if (status := await SyncStore(repository.prisma_client, "roi_observed").status()) and status.running:
raise HTTPException(409, "Cancel the running sync before changing the connection.")
original: Final = await load_stored_settings(repository)
target_id: Final = connection_id(patch.source_provider, patch.api_url)
selected_id: Final = patch.connection_id or target_id
selected: Final = next((entry for entry in stored_connections(original) if entry.id == selected_id), None)
if patch.connection_id and selected is None:
raise HTTPException(404, "This connection no longer exists. Reload Connections.")
if (
patch.connection_id
and selected_id != target_id
and any(entry.id == target_id for entry in stored_connections(original))
):
raise HTTPException(409, "This provider and host are already connected. Edit that connection instead.")
if patch.token is None and selected:
await connected_settings(repository, transport, selected_id)
refreshed: Final = await load_stored_settings(repository)
saved_connection: Final = next((entry for entry in stored_connections(refreshed) if entry.id == selected_id), None)
stored: Final = select_connection(refreshed, saved_connection) if saved_connection else refreshed
current: Final = await load_settings(repository, stored)
changed: Final = target_id != connection_id(current.source_provider, current.source_api_url)
existing_token: Final = current.gitlab_token if patch.source_provider == "gitlab" else current.github_token
token: Final = patch.token if patch.token is not None else "" if changed else existing_token.get_secret_value()
updates: Final[Mapping[str, object]] = {
"report_mode": "observed",
"source_provider": patch.source_provider,
"repos": patch.repos,
"update_interval_minutes": patch.update_interval_minutes
if patch.update_interval_minutes is not None
else current.update_interval_minutes,
"identity_map": {} if changed else current.identity_map,
"ignored_logins": () if changed else current.ignored_logins,
"connection_type": "token" if patch.token is not None or changed else current.connection_type,
"oauth_refresh_token": SecretStr("") if patch.token is not None or changed else current.oauth_refresh_token,
"oauth_expires_at": None if patch.token is not None or changed else current.oauth_expires_at,
("gitlab_api_url" if patch.source_provider == "gitlab" else "github_api_url"): patch.api_url,
("gitlab_token" if patch.source_provider == "gitlab" else "github_token"): SecretStr(token),
}
try:
settings: Final = ROISettings.model_validate({**current.model_dump(), **updates})
except ValidationError as exc:
raise HTTPException(422, exc.errors(include_context=False, include_input=False)) from None
source: Final = create_source(settings, transport)
try:
if settings.repos:
await source.test_repositories(settings.repos)
elif token:
await source.repositories(page=1)
except SourceError as exc:
raise HTTPException(502, str(exc)) from None
finally:
await source.close()
encrypted: Final = TypeAdapter(str).validate_python(encrypt_value_helper(token)) if token else ""
await save_settings(
repository,
settings,
encrypted if settings.source_provider == "github" else stored.github_token,
stored.estimator_key,
encrypted if settings.source_provider == "gitlab" else stored.gitlab_token,
revision=stored.revision,
replace_connection_id=patch.connection_id,
)
return await workspace_settings(repository)
@router.get("/report", response_model=ObservedReportResponse)
async def get_observed_report(
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ObservedReportResponse:
stored: Final = await load_stored_settings(repository)
data: Final = await _data(repository)
return ObservedReportResponse(report=summarize_workspace(data, stored_connections(stored)) if data else None)
@router.get("/identities", response_model=ObservedIdentities)
async def get_observed_identities(
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ObservedIdentities:
stored: Final = await load_stored_settings(repository)
data: Final = await _data(repository)
report: Final = summarize_workspace(data, stored_connections(stored)) if data else None
return ObservedIdentities(
gateway_emails=tuple(sorted(await read_gateway_user_emails(spend_prisma_client(repository.prisma_client)))),
identity_map=stored.identity_map,
unmatched_logins=report.unmatched_logins if report else (),
connections=tuple(
ObservedConnectionIdentities(
id=entry.id,
source_provider=entry.source_provider,
api_url=entry.api_url,
repos=entry.repos,
identity_map=entry.identity_map,
unmatched_logins=tuple(
login.split(":", 1)[-1] for login in report.unmatched_logins if login.startswith(entry.id + ":")
)
if report
else (),
)
for entry in stored_connections(stored)
),
)
@router.put("/identities", response_model=ObservedReportResponse)
async def save_observed_identities(
patch: ObservedIdentityUpdate,
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
) -> ObservedReportResponse:
email: Final = normalize_email(patch.email)
emails: Final = await read_gateway_user_emails(spend_prisma_client(repository.prisma_client))
if email not in emails:
raise HTTPException(422, "Choose an existing internal user email.")
stored: Final = await load_stored_settings(repository)
connections: Final = stored_connections(stored)
if not connections:
raise HTTPException(409, "Connect a repository before linking accounts.")
accounts: Final = (
patch.accounts
if patch.accounts is not None
else tuple(ObservedAccount(connection_id=active_connection(stored).id, login=login) for login in patch.logins)
)
if any(account.connection_id not in {entry.id for entry in connections} for account in accounts):
raise HTTPException(422, "Choose an existing connection.")
data: Final = await _data(repository)
report: Final = summarize_workspace(data, connections) if data else None
existing: Final = next((person.accounts for person in report.people if person.email == email), ()) if report else ()
def update(entry: StoredConnection) -> StoredConnection:
if patch.accounts is None and entry.id != active_connection(stored).id:
return entry
try:
logins: Final = tuple(
normalize_source_login(account.login, entry.source_provider)
for account in accounts
if account.connection_id == entry.id
)
except ValueError as exc:
raise HTTPException(422, str(exc)) from None
if any(login in entry.identity_map and entry.identity_map[login] != email for login in logins):
raise HTTPException(409, "An account is already linked to another email. Unlink it first.")
old: Final = frozenset(
(
*(login for login, address in entry.identity_map.items() if address == email),
*(account.login for account in existing if account.connection_id == entry.id),
)
)
return entry.model_copy(
update={
"identity_map": {
**{login: address for login, address in entry.identity_map.items() if address != email},
**dict.fromkeys(logins, email),
},
"ignored_logins": tuple(sorted((frozenset(entry.ignored_logins) | old) - frozenset(logins))),
}
)
updated: Final = tuple(update(entry) for entry in connections)
await save_connection_identities(repository, stored, updated)
return ObservedReportResponse(report=summarize_workspace(data, updated) if data else None)
@router.get("/sync", response_model=ROISyncStatus)
async def get_observed_sync(
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
manager: Annotated[ObservedSyncManager, Depends(get_observed_manager)],
) -> ROISyncStatus:
return await SyncStore(repository.prisma_client, "roi_observed").status() or manager.status
async def _report_days(repository: ConfigRepository, days: int | None) -> int:
if days is not None:
return days
saved: Final = await repository.get_param(_REPORT_KEY)
if saved is None:
return 28
try:
data: Final = ObservedData.model_validate(saved.param_value)
except ValidationError:
return 28
return (data.current.window.end - data.current.window.start).days + 1
async def _start_sync(
repository: ConfigRepository,
manager: ObservedSyncManager,
transport: httpx.AsyncBaseTransport | None,
scheduled_interval: float = 0,
days: int | None = None,
) -> bool:
from litellm.proxy.management_endpoints.roi_calculator_endpoints import branch_spend_reader
stored: Final = await load_stored_settings(repository)
reporting_days: Final = await _report_days(repository, days)
entries: Final = tuple(entry for entry in stored_connections(stored) if entry.repos)
if not entries:
raise HTTPException(409, "Select at least one repository.")
settings: Final = tuple([await connected_settings(repository, transport, entry.id) for entry in entries])
await enable_observed_reporting(repository)
async def build(progress: Progress) -> ObservedData:
from litellm.proxy.management_endpoints.roi_calculator_endpoints import gateway_user_reader, spend_reader
data: Final = await collect_workspace(
tuple((entry, branch_spend_reader(repository, entry)) for entry in settings),
spend_reader(repository),
gateway_user_reader(repository),
datetime.now(timezone.utc),
progress,
transport,
days=reporting_days,
)
current: Final = tuple(
entry for entry in stored_connections(await load_stored_settings(repository)) if entry.repos
)
if tuple((entry.id, entry.repos) for entry in current) != tuple((entry.id, entry.repos) for entry in entries):
raise SourceError("The connection changed during sync. Sync again with the current repositories.")
return data
return await manager.start(build, SyncStore(repository.prisma_client, "roi_observed"), scheduled_interval)
@router.post("/sync", response_model=ROISyncStatus, status_code=202)
async def start_observed_sync(
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
manager: Annotated[ObservedSyncManager, Depends(get_observed_manager)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)],
days: Annotated[int | None, Query(ge=1, le=366)] = None,
) -> ROISyncStatus:
if not await _start_sync(repository, manager, transport, days=days):
raise HTTPException(409, "A sync is already running.")
return manager.status
@router.delete("/sync", response_model=ROISyncStatus)
async def cancel_observed_sync(
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
manager: Annotated[ObservedSyncManager, Depends(get_observed_manager)],
) -> ROISyncStatus:
store: Final = SyncStore(repository.prisma_client, "roi_observed")
await store.cancel()
await manager.cancel()
return await store.status() or manager.status
async def run_observed_schedule() -> None:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return
repository: Final = ConfigRepository(prisma_client, use_writer=True)
settings: Final = await load_settings(repository)
if (
not any(entry.repos for entry in stored_connections(await load_stored_settings(repository)))
or not settings.update_interval_minutes
):
return
status: Final = await SyncStore(prisma_client, "roi_observed").status()
if status and status.running:
return
anchor: Final = status.finished_at if status else None
if anchor and datetime.fromisoformat(anchor) + timedelta(minutes=settings.update_interval_minutes) > datetime.now(
timezone.utc
):
return
await _start_sync(repository, _MANAGER, None, settings.update_interval_minutes)
@router.get("/repositories", response_model=ROIRepositoriesResponse)
async def observed_repositories(
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)],
connection: Annotated[str | None, Query(max_length=100)] = None,
query: Annotated[str, Query(max_length=200)] = "",
page: Annotated[int, Query(ge=1, le=1000)] = 1,
) -> ROIRepositoriesResponse:
source: Final = create_source(await connected_settings(repository, transport, connection), transport)
try:
repositories, more = await source.repositories(query, page)
except SourceError as exc:
raise HTTPException(502, str(exc)) from None
finally:
await source.close()
return ROIRepositoriesResponse(
repositories=tuple(
ROIRepository(name=name, visibility=visibility, archived=archived)
for name, visibility, archived in repositories
),
page=page,
has_more=more,
)
@router.get("/apps", response_model=ObservedApps)
async def observed_apps(_user: Annotated[UserAPIKeyAuth, Depends(read_admin)]) -> ObservedApps:
def details(provider: Provider) -> ObservedApp:
config: Final = oauth_config(provider)
return ObservedApp(
configured=config is not None,
can_install=bool(config and config.installation_url),
api_url=config.api_url if config else None,
callback_url=config.redirect_uri if config else None,
)
return ObservedApps(github=details("github"), gitlab=details("gitlab"))
@router.post("/oauth/{provider}/start", response_model=ObservedAuthorization)
async def start_observed_authorization(
provider: Provider,
_user: Annotated[UserAPIKeyAuth, Depends(write_admin)],
repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)],
install: bool = False,
) -> JSONResponse:
config: Final = oauth_config(provider)
if config is None:
raise HTTPException(409, "Configure the provider app client ID, client secret, and PROXY_BASE_URL first.")
if install and not config.installation_url:
raise HTTPException(409, "Configure the GitHub app slug to manage repository access.")
url, nonce = await begin_authorization(repository, config, install=install)
response: Final = JSONResponse(ObservedAuthorization(url=url).model_dump(mode="json"))
response.set_cookie(
"litellm_roi_oauth",
nonce,
httponly=True,
secure=config.proxy_url.startswith("https://"),
samesite="lax",
max_age=600,
path=config.cookie_path,
)
response.headers["Cache-Control"] = "no-store"
return response
def get_oauth_repository() -> ConfigRepository:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(503, "The database is unavailable. Try connecting again later.")
return ConfigRepository(prisma_client, use_writer=True)
def _authorization_redirect(config: OAuthConfig, query: str) -> RedirectResponse:
response: Final = RedirectResponse(config.proxy_url + "/ui/roi-calculator/?" + query, status_code=303)
response.delete_cookie("litellm_roi_oauth", path=config.cookie_path)
response.headers["Cache-Control"] = "no-store"
response.headers["Referrer-Policy"] = "no-referrer"
return response
@router.get("/oauth/{provider}/callback", include_in_schema=False)
async def observed_authorization_callback(
provider: Provider,
request: Request,
repository: Annotated[ConfigRepository, Depends(get_oauth_repository)],
transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)],
state: str = "",
code: str = "",
error: str = "",
) -> RedirectResponse:
config: Final = oauth_config(provider)
if config is None:
raise HTTPException(409, "The provider app is not configured.")
try:
return await _complete_authorization(config, request, repository, transport, state, code, error)
except HTTPException:
return _authorization_redirect(config, "connection_failed=1")
async def _complete_authorization(
config: OAuthConfig,
request: Request,
repository: ConfigRepository,
transport: httpx.AsyncBaseTransport | None,
state: str,
code: str,
error: str,
) -> RedirectResponse:
verified: Final = await consume_state(repository, state, request.cookies.get("litellm_roi_oauth", ""), config)
if error or not code:
return _authorization_redirect(config, "connection_cancelled=1")
if (status := await SyncStore(repository.prisma_client, "roi_observed").status()) and status.running:
raise HTTPException(409, "Cancel the running sync before changing the connection.")
grant: Final = await exchange_code(config, verified, code, transport)
validation_settings: Final = ROISettings.model_validate(
{
"source_provider": config.provider,
"connection_type": "app",
("github_api_url" if config.provider == "github" else "gitlab_api_url"): config.api_url,
("github_token" if config.provider == "github" else "gitlab_token"): grant.access_token,
}
)
source: Final = create_source(validation_settings, transport)
try:
await source.repositories(page=1)
except SourceError as exc:
raise HTTPException(502, str(exc)) from None
finally:
await source.close()
await save_grant(repository, config, grant, revision=verified.settings_revision)
return _authorization_redirect(config, "connected=" + config.provider)
@router.get("/oauth/github/installed", include_in_schema=False)
async def observed_installation_callback(
request: Request,
repository: Annotated[ConfigRepository, Depends(get_oauth_repository)],
state: str = "",
) -> RedirectResponse:
config: Final = oauth_config("github")
if config is None or config.installation_url is None:
raise HTTPException(409, "The GitHub app is not configured.")
try:
verified: Final = await consume_state(
repository, state, request.cookies.get("litellm_roi_oauth", ""), config, flow="install"
)
if (await load_stored_settings(repository)).revision != verified.settings_revision:
raise HTTPException(409, "The connection changed during installation. Start again from Connections.")
url, nonce = await begin_authorization(repository, config)
except HTTPException:
return _authorization_redirect(config, "connection_failed=1")
response: Final = RedirectResponse(url, status_code=303)
response.set_cookie(
"litellm_roi_oauth",
nonce,
httponly=True,
secure=config.proxy_url.startswith("https://"),
samesite="lax",
max_age=600,
path=config.cookie_path,
)
response.headers["Cache-Control"] = "no-store"
response.headers["Referrer-Policy"] = "no-referrer"
return response

52
litellm/proxy/roi_calculator/README.md Normal file
View file

@ -0,0 +1,52 @@
# ROI Calculator
The dashboard compares merged pull or merge requests, elapsed time from opening to merge, new bug and regression issues, and recorded gateway spend over 7, 28, or 90 complete UTC days. Compare against the immediately preceding period of the same length or the same-length period last year
The calculator combines repository activity with spend recorded by the gateway. Spend per merged change is a person's recorded gateway spend during the period divided by their matched merged changes. To track a branch's AI cost, send repository and branch tags with each request
## Connect repositories
Use **Preview sample report** beside the title to explore the dashboard before connecting repositories. Sample periods, engineer details, quality signals, and branch spend work without changing your connections or live report. **Exit demo** returns to your report or setup
Open `/ui/roi-calculator/`, choose GitHub or GitLab, then connect with an app or access token. Select several repositories and start the sync. Use **Add connection** to keep both providers connected. Each provider and API host retains its credentials, repositories, and identity mappings, and the report combines their activity while counting each person’s gateway spend once. Public repositories also accept an empty token, subject to the provider's anonymous API limits
For GitHub tokens, grant read access to metadata, pull requests and issues. GitLab tokens require `read_api`. Self-hosted instances use their API URL, for example `https://git.example.com/api/v4`
## Configure app authorization
Register a GitHub App with read-only repository permissions for metadata, pull requests and issues. Enable expiring user access tokens and leave authorization during installation disabled, since the gateway starts authorization after installation. Generate a private key in the app settings to allow installation and store it securely. The gateway uses a generated client secret for authorization and does not need the private key
Register a confidential GitLab OAuth application with `read_api` and `read_user` scopes
Set `PROXY_BASE_URL` to the gateway's public URL. The callback URLs are `<PROXY_BASE_URL>/roi-calculator/observed/oauth/github/callback` and `<PROXY_BASE_URL>/roi-calculator/observed/oauth/gitlab/callback`
Set the GitHub App setup URL to `<PROXY_BASE_URL>/roi-calculator/observed/oauth/github/installed`, enable **Redirect on update**, and set `LITELLM_ROI_GITHUB_APP_SLUG` to its URL slug. The first connection then starts with repository installation and continues to user authorization
Set `LITELLM_ROI_GITHUB_CLIENT_ID` and `LITELLM_ROI_GITHUB_CLIENT_SECRET` for GitHub, or `LITELLM_ROI_GITLAB_CLIENT_ID` and `LITELLM_ROI_GITLAB_CLIENT_SECRET` for GitLab. For a self-hosted provider, set `LITELLM_ROI_GITHUB_URL` or `LITELLM_ROI_GITLAB_URL` to its base URL without the API suffix
The gateway encrypts access and refresh tokens using its configured encryption key. Authorization uses PKCE and an expiring, single-use state tied to an HTTP-only browser cookie. Refreshes are coordinated across gateway workers
## Link people
Use **Link accounts** to associate several current or historical usernames with one internal email. Each connection has a separate username field, so a GitHub username never matches a GitLab user implicitly. Saving immediately recalculates the report without fetching repositories again. Public profile emails match automatically when they resolve unambiguously to an internal user
Agent-authored changes count for a person only when the supported agent metadata explicitly names a requester. Repository issue counts and revert titles are quality signals, not an individual defect score
Bug and regression counts combine repositories with issue tracking enabled. They remain unavailable when none of the selected repositories has issue tracking enabled
## Sync behavior
The default refresh interval is daily and applies to every connection in the workspace. Adding or editing a connection preserves it unless `update_interval_minutes` is supplied. The observed settings API accepts `update_interval_minutes: 0` for manual updates. A cancelled or failed sync preserves the last complete report
Existing settings retain `report_mode: legacy` and their scheduled reports until an administrator saves a connection, authorizes an app, or starts an observed sync. Reading the new dashboard alone does not change the mode. The legacy settings API can explicitly select `report_mode: legacy` again
GitHub collection splits large searches into smaller date ranges to avoid its search-result limit. Both providers validate pagination and reject incomplete responses instead of publishing partial counts
## Branch request tags
Send `repo:github.com/owner/repo` or `repo:gitlab.com/group/project` together with `branch:feature/name` in `metadata.tags`, top-level `tags`, or the comma-separated `x-litellm-tags` header. The tags must identify the source repository and branch, including forks
The report sums recorded requests inside its UTC dates. A branch cost is assigned to a merged change only when that source branch matches one change in the period. Reused branches stay visible in Branch spend without duplicating costs across changes. No retained tagged requests means unknown cost; a recorded zero remains zero
An empty repository produces a successful report with zero merged changes and no merge duration or spend-per-change ratio

21
litellm/proxy/roi_calculator/branch_spend.py
View file

@ -61,12 +61,23 @@ async def read_branch_spend(
def attribute_branches(
pulls: tuple[ROIPullRecord, ...], spend: tuple[ROIBranchSpend, ...] | None
) -> Mapping[tuple[str, int], ROIBranchAttribution]:
counts: Final = Counter((pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in pulls)
return attribute_branch_keys(
tuple(
(pull["repo"], pull["number"], pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in pulls
),
spend,
)
def attribute_branch_keys(
pulls: tuple[tuple[str, int, str, str], ...], spend: tuple[ROIBranchSpend, ...] | None
) -> Mapping[tuple[str, int], ROIBranchAttribution]:
counts: Final = Counter((pull[2], pull[3]) for pull in pulls)
costs: Final = {(row.repo, row.branch): row for row in spend or ()}
def attribute(pull: ROIPullRecord) -> ROIBranchAttribution:
repo: Final = pull.get("source_repo", "")
branch: Final = pull.get("source_branch", "")
def attribute(pull: tuple[str, int, str, str]) -> ROIBranchAttribution:
repo: Final = pull[2]
branch: Final = pull[3]
cost: Final = costs.get((repo, branch))
if spend is None:
return ROIBranchAttribution(repo=repo, branch=branch, status="unavailable")
@ -78,4 +89,4 @@ def attribute_branches(
repo=repo, branch=branch, spend=cost.spend, requests=cost.requests, status="matched"
)
return {(pull["repo"], pull["number"]): attribute(pull) for pull in pulls}
return {(pull[0], pull[1]): attribute(pull) for pull in pulls}

74
litellm/proxy/roi_calculator/github.py
View file

@ -1,6 +1,6 @@
import asyncio
from collections.abc import AsyncIterator, Mapping
from datetime import date
from datetime import date, datetime
from types import MappingProxyType
from typing import Final, TypeVar
from urllib.parse import quote
@ -15,6 +15,7 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.proxy.roi_calculator.analytics import normalize_email
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.roi_calculator import ROIPullCommit, ROIPullEvidence, ROIPullFile, ROISettings
from litellm.types.roi_observed import ObservedIssue
_T: Final = TypeVar("_T")
@ -29,6 +30,8 @@ class _GitHubModel(BaseModel):
class _GitHubUser(_GitHubModel):
login: str | None = None
type: str = "User"
email: str | None = None
class _GitHubHeadRepository(_GitHubModel):
@ -50,6 +53,7 @@ class GitHubPullListItem(_GitHubModel):
body: str | None = None
head: _GitHubHead | None = None
user: _GitHubUser | None = None
created_at: datetime | None = None
class _RepositoryItem(_GitHubModel):
@ -215,15 +219,18 @@ _GRAPHQL_QUERY: Final = """query($owner:String!, $name:String!, $number:Int!, $c
}"""
async def _request(
async def request_github(
client: httpx.AsyncClient,
method: str,
path: str,
params: Mapping[str, str | int] | None = None,
json_body: object | None = None,
headers: Mapping[str, str] | None = None,
*,
read_only: bool = False,
) -> httpx.Response:
async def send(attempt: int) -> httpx.Response:
retryable: Final = (method == "GET" or read_only) and attempt < 2
try:
response: Final = await client.request(
method,
@ -233,8 +240,11 @@ async def _request(
headers=headers,
)
except httpx.RequestError:
if retryable:
await asyncio.sleep(0.5 * (attempt + 1))
return await send(attempt + 1)
raise SourceError("Could not reach GitHub. Check the API URL and network connection.") from None
if response.status_code in (429, 502, 503, 504) and method == "GET" and attempt < 2:
if response.status_code in (429, 502, 503, 504) and retryable:
await asyncio.sleep(0.5 * (attempt + 1))
return await send(attempt + 1)
if response.status_code >= 400:
@ -267,7 +277,7 @@ async def _fetch_page(
headers: Mapping[str, str] | None = None,
error_message: str = "GitHub returned an unexpected pagination response.",
) -> tuple[tuple[_T, ...], bool]:
response: Final = await _request(
response: Final = await request_github(
client,
"GET",
path,
@ -312,6 +322,21 @@ class _GitHubUserProfile(_GitHubModel):
email: str | None = None
class _IssueLabel(_GitHubModel):
name: str
class _Issue(_GitHubModel):
number: int
created_at: datetime
labels: tuple[_IssueLabel, ...] = ()
pull_request: object | None = None
class GitHubIssueSettings(BaseModel):
has_issues: bool
class GitHub:
def __init__(
self,
@ -408,8 +433,8 @@ class GitHub:
async def test_repositories(self, repos: tuple[str, ...]) -> None:
for repo in repos:
await _request(self.client, "GET", self._url(f"repos/{repo}"), headers=self._headers)
await _request(
await request_github(self.client, "GET", self._url(f"repos/{repo}"), headers=self._headers)
await request_github(
self.client,
"GET",
self._url(f"repos/{repo}/pulls"),
@ -438,8 +463,40 @@ class GitHub:
return await _collect(matching_pulls())
async def issues(self, repo: str, start: date, end: date) -> tuple[ObservedIssue, ...] | None:
response: Final = await request_github(self.client, "GET", self._url(f"repos/{repo}"), headers=self._headers)
try:
settings: Final = GitHubIssueSettings.model_validate(response.json())
except ValueError:
raise SourceError("GitHub returned invalid repository settings.") from None
if not settings.has_issues:
return None
async def matching_issues() -> AsyncIterator[ObservedIssue]:
async for page in _pages(
self.client,
self._url(f"repos/{repo}/issues"),
TypeAdapter(tuple[_Issue, ...]),
MappingProxyType(
{"state": "all", "sort": "created", "direction": "desc", "since": f"{start}T00:00:00Z"}
),
headers=self._headers,
):
for issue in page:
if issue.pull_request is None and start <= issue.created_at.date() <= end:
yield ObservedIssue(
repo=repo,
number=issue.number,
created_at=issue.created_at,
labels=tuple(label.name for label in issue.labels),
)
if page and page[-1].created_at.date() < start:
return
return await _collect(matching_issues())
async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence:
detail_response: Final = await _request(
detail_response: Final = await request_github(
self.client,
"GET",
self._url(f"repos/{repo}/pulls/{pull.number}"),
@ -574,7 +631,7 @@ class GitHub:
) -> tuple[tuple[ROIPullCommit, ...], tuple[tuple[str, str], ...], int]:
if remaining_pages == 0:
raise SourceError("GitHub commit pagination limit was reached.")
response: Final = await _request(
response: Final = await request_github(
self.client,
"POST",
endpoint,
@ -583,6 +640,7 @@ class GitHub:
query=_GRAPHQL_QUERY,
variables=_GraphQLVariables(owner=owner, name=name, number=number, cursor=cursor),
),
read_only=True,
)
try:
parsed: Final = _GRAPHQL_RESPONSE.validate_python(response.json())

196
litellm/proxy/roi_calculator/github_observed.py Normal file
View file

@ -0,0 +1,196 @@
from datetime import date, datetime, time, timedelta, timezone
from typing import Final, Literal
import httpx
from pydantic import BaseModel, Field
from litellm.proxy.roi_calculator.github import GitHubIssueSettings, GitHubPullListItem, SourceError, request_github
from litellm.types.roi_calculator import ROISettings
from litellm.types.roi_observed import ObservedIssue
class _PageInfo(BaseModel):
hasNextPage: bool = False
endCursor: str | None = None
class _Author(BaseModel):
login: str
kind: str = Field(alias="__typename")
email: str | None = None
class _Repository(BaseModel):
nameWithOwner: str
class _Label(BaseModel):
name: str
class _Labels(BaseModel):
nodes: tuple[_Label, ...] = ()
pageInfo: _PageInfo = Field(default_factory=_PageInfo)
class _Node(BaseModel):
number: int
url: str
title: str
createdAt: datetime
updatedAt: str
mergedAt: str | None = None
author: _Author | None = None
body: str = ""
headRefName: str = ""
headRefOid: str = ""
headRepository: _Repository | None = None
labels: _Labels = Field(default_factory=_Labels)
def pull(self) -> GitHubPullListItem:
return GitHubPullListItem.model_validate(
{
"number": self.number,
"html_url": self.url,
"title": self.title,
"body": self.body,
"created_at": self.createdAt,
"merged_at": self.mergedAt,
"updated_at": self.updatedAt,
"user": {"login": self.author.login, "type": self.author.kind, "email": self.author.email}
if self.author
else None,
"head": {
"ref": self.headRefName,
"sha": self.headRefOid,
"repo": {"full_name": self.headRepository.nameWithOwner} if self.headRepository else None,
},
}
)
class _Search(BaseModel):
issueCount: int
pageInfo: _PageInfo
nodes: tuple[_Node, ...]
class _Data(BaseModel):
search: _Search
class _Response(BaseModel):
data: _Data | None = None
errors: tuple[object, ...] = ()
_QUERY: Final = """query($q:String!, $after:String) {
search(query:$q, type:ISSUE, first:100, after:$after) {
issueCount pageInfo { hasNextPage endCursor }
nodes {
... on PullRequest {
number url title body createdAt updatedAt mergedAt headRefName headRefOid
author { __typename login ... on User { email } } headRepository { nameWithOwner }
}
... on Issue {
number url title createdAt updatedAt
labels(first:100) { nodes { name } pageInfo { hasNextPage } }
}
}
}
}"""
class GitHubObserved:
def __init__(self, settings: ROISettings, client: httpx.AsyncClient) -> None:
self._client: Final = client
self._api_url: Final = settings.github_api_url
self._url: Final = (
"https://api.github.com/graphql"
if settings.github_api_url == "https://api.github.com"
else settings.github_api_url.removesuffix("/api/v3") + "/api/graphql"
)
self._headers: Final = {"Authorization": "Bearer " + settings.github_token.get_secret_value()}
async def _page(self, query: str, cursor: str | None = None) -> _Search:
response: Final = await request_github(
self._client,
"POST",
self._url,
headers=self._headers,
json_body={"query": _QUERY, "variables": {"q": query, "after": cursor}},
read_only=True,
)
try:
result: Final = _Response.model_validate(response.json())
except ValueError:
raise SourceError("GitHub returned invalid activity data. Try syncing again.") from None
if result.errors or result.data is None:
raise SourceError("GitHub could not read all activity. Check app permissions and rate limits, then retry.")
return result.data.search
async def _range(
self, repo: str, start: datetime, end: datetime, kind: Literal["pull", "issue"]
) -> tuple[_Node, ...]:
qualifier: Final = "merged" if kind == "pull" else "created"
source: Final = "is:pr is:merged" if kind == "pull" else "is:issue"
lower: Final = start.strftime("%Y-%m-%dT%H:%M:%SZ")
upper: Final = (end - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
query: Final = f"repo:{repo} {source} {qualifier}:{lower}..{upper} sort:created-asc"
first: Final = await self._page(query)
if first.issueCount > 1000:
seconds: Final = int((end - start).total_seconds())
if seconds < 2:
raise SourceError("GitHub has more than 1,000 results in one second. The report was not truncated.")
middle: Final = start + timedelta(seconds=seconds // 2)
left: Final = await self._range(repo, start, middle, kind)
return left + await self._range(repo, middle, end, kind)
async def remaining(page: _Search, seen: frozenset[str]) -> tuple[_Node, ...]:
if not page.pageInfo.hasNextPage:
return page.nodes
cursor: Final = page.pageInfo.endCursor
if not cursor or cursor in seen or len(seen) >= 10:
raise SourceError("GitHub returned incomplete pagination. The previous report was kept.")
following: Final = await self._page(query, cursor)
return page.nodes + await remaining(following, seen | {cursor})
nodes: Final = await remaining(first, frozenset())
if len(nodes) != first.issueCount or len(frozenset(node.url for node in nodes)) != len(nodes):
raise SourceError("GitHub activity changed during collection. Retry to get a complete report.")
return nodes
async def _read(self, repo: str, start: date, end: date, kind: Literal["pull", "issue"]) -> tuple[_Node, ...]:
return await self._range(
repo,
datetime.combine(start, time.min, timezone.utc),
datetime.combine(end + timedelta(days=1), time.min, timezone.utc),
kind,
)
async def pulls(self, repo: str, start: date, end: date) -> tuple[GitHubPullListItem, ...]:
nodes: Final = await self._read(repo, start, end, "pull")
return tuple(node.pull() for node in nodes)
async def issues(self, repo: str, start: date, end: date) -> tuple[ObservedIssue, ...] | None:
response: Final = await request_github(
self._client, "GET", f"{self._api_url}/repos/{repo}", headers=self._headers
)
try:
settings: Final = GitHubIssueSettings.model_validate(response.json())
except ValueError:
raise SourceError("GitHub returned invalid repository settings.") from None
if not settings.has_issues:
return None
nodes: Final = await self._read(repo, start, end, "issue")
if any(node.labels.pageInfo.hasNextPage for node in nodes):
raise SourceError("GitHub returned incomplete issue labels. The previous report was kept.")
return tuple(
ObservedIssue(
repo=repo,
number=node.number,
created_at=node.createdAt,
labels=tuple(label.name for label in node.labels.nodes),
)
for node in nodes
)

47
litellm/proxy/roi_calculator/gitlab.py
View file

@ -1,6 +1,6 @@
import asyncio
from collections.abc import Mapping
from datetime import date
from datetime import date, datetime, timedelta
from types import MappingProxyType
from typing import Final, TypeVar
from urllib.parse import quote
@ -16,6 +16,7 @@ from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError
from litellm.proxy.roi_calculator.source import repository_tag
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.roi_calculator import ROIPullCommit, ROIPullEvidence, ROIPullFile, ROISettings
from litellm.types.roi_observed import ObservedIssue
_T: Final = TypeVar("_T", bound=BaseModel)
@ -23,6 +24,7 @@ _T: Final = TypeVar("_T", bound=BaseModel)
class _User(BaseModel):
username: str
public_email: str | None = None
bot: bool = False
class _Project(BaseModel):
@ -30,6 +32,8 @@ class _Project(BaseModel):
path_with_namespace: str
visibility: str = "private"
archived: bool = False
issues_enabled: bool = True
issues_access_level: str = "enabled"
class _MergeRequest(BaseModel):
@ -40,6 +44,7 @@ class _MergeRequest(BaseModel):
author: _User
merged_at: str | None
updated_at: str
created_at: datetime | None = None
sha: str | None = None
source_branch: str
source_project_id: int | None
@ -52,7 +57,8 @@ class _MergeRequest(BaseModel):
"title": self.title,
"body": self.description or "",
"html_url": self.web_url,
"user": {"login": self.author.username},
"user": {"login": self.author.username, "type": "Bot" if self.author.bot else "User"},
"created_at": self.created_at,
"merged_at": self.merged_at,
"updated_at": self.updated_at,
"head": {
@ -94,11 +100,20 @@ class _Commit(BaseModel):
message: str
class _Issue(BaseModel):
iid: int
created_at: datetime
labels: tuple[str, ...] = ()
class GitLab:
def __init__(self, settings: ROISettings, transport: httpx.AsyncBaseTransport | None = None) -> None:
self.settings: Final = settings
token: Final = settings.gitlab_token.get_secret_value()
self.headers: Final = {"Accept": "application/json", **({"PRIVATE-TOKEN": token} if token else {})}
authorization: Final = (
{"Authorization": f"Bearer {token}"} if settings.connection_type == "app" else {"PRIVATE-TOKEN": token}
)
self.headers: Final = {"Accept": "application/json", **(authorization if token else {})}
self.client: Final = get_async_httpx_client(
llm_provider=httpxSpecialProvider.ROICalculator,
params={"timeout": 45, "follow_redirects": False, "transport": transport},
@ -171,7 +186,9 @@ class GitLab:
params: Final = {
"simple": "true",
"search": query,
**({"membership": "true"} if self.headers.get("PRIVATE-TOKEN") else {}),
**(
{"membership": "true"} if self.headers.get("PRIVATE-TOKEN") or self.headers.get("Authorization") else {}
),
}
items, more = await self._page("projects", _Project, params, page)
return tuple((item.path_with_namespace, item.visibility, item.archived) for item in items), more
@ -193,6 +210,8 @@ class GitLab:
"state": "merged",
"scope": "all",
"updated_after": start.isoformat() + "T00:00:00Z",
"merged_after": start.isoformat() + "T00:00:00Z",
"merged_before": (end + timedelta(days=1)).isoformat() + "T00:00:00Z",
"order_by": "updated_at",
"sort": "desc",
},
@ -218,6 +237,26 @@ class GitLab:
self.profiles = MappingProxyType({**self.profiles, login.casefold(): email})
return email
async def issues(self, repo: str, start: date, end: date) -> tuple[ObservedIssue, ...] | None:
project: Final = await self._project(repo)
if not project.issues_enabled or project.issues_access_level == "disabled":
return None
issues: Final = await self._all(
f"projects/{project.id}/issues",
_Issue,
{
"scope": "all",
"state": "all",
"created_after": f"{start}T00:00:00Z",
"created_before": f"{end + timedelta(days=1)}T00:00:00Z",
},
)
return tuple(
ObservedIssue(repo=repo, number=issue.iid, created_at=issue.created_at, labels=issue.labels)
for issue in issues
if start <= issue.created_at.date() <= end
)
async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence:
project: Final = await self._project(repo)
path: Final = f"projects/{project.id}/merge_requests/{pull.number}"

401
litellm/proxy/roi_calculator/oauth.py Normal file
View file

@ -0,0 +1,401 @@
import asyncio
import hashlib
import json
import os
import re
import secrets
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Final, Literal, Protocol, TypeAlias, cast
from urllib.parse import parse_qsl, urlencode, urlsplit
import httpx
from fastapi import HTTPException
from oauthlib.oauth2 import WebApplicationClient
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params
)
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper
from litellm.proxy.roi_calculator.settings import (
active_connection,
connection_id,
load_settings,
load_stored_settings,
save_settings,
select_connection,
stored_connections,
)
from litellm.proxy.roi_calculator.sync_store import SyncStore
from litellm.repositories.config_repository import ConfigRepository
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.roi_calculator import ROISettings, ROISyncStatus
Provider: TypeAlias = Literal["github", "gitlab"]
_STATE_PREFIX: Final = "roi_oauth_state_"
@dataclass(frozen=True, slots=True)
class OAuthConfig:
provider: Provider
api_url: str
base_url: str
client_id: str
client_secret: SecretStr
proxy_url: str
app_slug: str = ""
@property
def cookie_path(self) -> str:
return urlsplit(self.proxy_url).path + "/roi-calculator/observed/oauth"
@property
def installation_url(self) -> str | None:
if self.provider != "github" or not self.app_slug:
return None
path: Final = "apps" if self.api_url == "https://api.github.com" else "github-apps"
return f"{self.base_url}/{path}/{self.app_slug}/installations/new"
@property
def redirect_uri(self) -> str:
return f"{self.proxy_url}/roi-calculator/observed/oauth/{self.provider}/callback"
@property
def authorize_url(self) -> str:
return self.base_url + ("/login/oauth/authorize" if self.provider == "github" else "/oauth/authorize")
@property
def token_url(self) -> str:
return self.base_url + ("/login/oauth/access_token" if self.provider == "github" else "/oauth/token")
def oauth_config(provider: Provider) -> OAuthConfig | None:
prefix: Final = f"LITELLM_ROI_{provider.upper()}_"
client_id: Final = os.environ.get(prefix + "CLIENT_ID", "")
client_secret: Final = os.environ.get(prefix + "CLIENT_SECRET", "")
proxy_url: Final = os.environ.get("PROXY_BASE_URL", "").rstrip("/")
if not client_id or not client_secret or not proxy_url:
return None
try:
parsed: Final = urlsplit(proxy_url)
except ValueError:
return None
if parsed.username or parsed.password or parsed.query or parsed.fragment or not parsed.hostname:
return None
if parsed.scheme != "https" and not (parsed.scheme == "http" and parsed.hostname in ("localhost", "127.0.0.1")):
return None
base: Final = os.environ.get(
prefix + "URL", "https://github.com" if provider == "github" else "https://gitlab.com"
).rstrip("/")
api_url: Final = (
"https://api.github.com"
if base == "https://github.com"
else base + ("/api/v3" if provider == "github" else "/api/v4")
)
try:
validated: Final = ROISettings.model_validate(
{
"source_provider": provider,
("github_api_url" if provider == "github" else "gitlab_api_url"): api_url,
}
)
except ValueError:
return None
app_slug: Final = os.environ.get(prefix + "APP_SLUG", "")
if app_slug and not re.fullmatch(r"[A-Za-z0-9-]+", app_slug):
return None
return OAuthConfig(
provider, validated.source_api_url, base, client_id, SecretStr(client_secret), proxy_url, app_slug
)
class OAuthState(BaseModel):
model_config = ConfigDict(frozen=True)
provider: Provider
client_id: str
api_url: str
browser_nonce: SecretStr
verifier: SecretStr
expires_at: datetime
settings_revision: int
flow: Literal["authorize", "install"] = "authorize"
class _Envelope(BaseModel):
payload: str
class _StateRow(BaseModel):
param_value: _Envelope
class _Database(Protocol):
async def query_raw(self, query: str, *args: object) -> object: ...
async def execute_raw(self, query: str, *args: object) -> int: ...
def _state_key(state: str) -> str:
return _STATE_PREFIX + hashlib.sha256(state.encode()).hexdigest()
async def begin_authorization(
repository: ConfigRepository, config: OAuthConfig, *, install: bool = False
) -> tuple[str, str]:
client: Final = WebApplicationClient(config.client_id)
verifier: Final = TypeAdapter(str).validate_python(client.create_code_verifier(64))
challenge: Final = TypeAdapter(str).validate_python(client.create_code_challenge(verifier, "S256"))
state: Final = secrets.token_urlsafe(32)
nonce: Final = secrets.token_urlsafe(32)
stored: Final = await load_stored_settings(repository)
value: Final = OAuthState(
settings_revision=stored.revision,
flow="install" if install else "authorize",
provider=config.provider,
client_id=config.client_id,
api_url=config.api_url,
browser_nonce=SecretStr(nonce),
verifier=SecretStr(verifier),
expires_at=datetime.now(timezone.utc) + timedelta(minutes=10),
)
encoded: Final = json.dumps({**value.model_dump(mode="json"), "browser_nonce": nonce, "verifier": verifier})
payload: Final = TypeAdapter(str).validate_python(encrypt_value_helper(encoded))
await repository.set_param(_state_key(state), _Envelope(payload=payload).model_dump(mode="json"))
delegate: Final = repository.prisma_client.writer_db
database: Final = cast(_Database, delegate) # cast-ok: Prisma delegates database methods dynamically
await database.execute_raw(
"DELETE FROM \"LiteLLM_Config\" WHERE starts_with(param_name, $1) AND last_run_at < NOW() - INTERVAL '20 minutes'",
_STATE_PREFIX,
)
url: Final = TypeAdapter(str).validate_python(
client.prepare_request_uri( # pyright: ignore[reportUnknownMemberType] # oauthlib leaves extension kwargs untyped
config.authorize_url,
redirect_uri=config.redirect_uri,
scope="read_api read_user" if config.provider == "gitlab" else None,
state=state,
code_challenge=challenge,
code_challenge_method="S256",
)
)
if install and config.installation_url:
return config.installation_url + "?" + urlencode({"state": state}), nonce
return url, nonce
async def consume_state(
repository: ConfigRepository,
state: str,
nonce: str,
config: OAuthConfig,
*,
flow: Literal["authorize", "install"] = "authorize",
) -> OAuthState:
if not state or not nonce or len(state) > 200:
raise HTTPException(400, "The connection expired. Start again from the ROI Calculator.")
delegate: Final = repository.prisma_client.writer_db
database: Final = cast(_Database, delegate) # cast-ok: Prisma delegates database methods dynamically
rows: Final = TypeAdapter(tuple[_StateRow, ...]).validate_python(
await database.query_raw(
'DELETE FROM "LiteLLM_Config" WHERE param_name = $1 RETURNING param_value',
_state_key(state),
)
)
if not rows:
raise HTTPException(400, "This connection was already used or expired. Start again.")
plaintext: Final = decrypt_value_helper(rows[0].param_value.payload, _state_key(state))
if plaintext is None:
raise HTTPException(400, "Could not verify the connection. Start again.")
value: Final = OAuthState.model_validate_json(plaintext)
if (
value.flow != flow
or not secrets.compare_digest(value.browser_nonce.get_secret_value(), nonce)
or value.expires_at < datetime.now(timezone.utc)
or (value.provider, value.client_id, value.api_url) != (config.provider, config.client_id, config.api_url)
):
raise HTTPException(400, "Could not verify the connection. Start again in the same browser.")
return value
class TokenGrant(BaseModel):
access_token: SecretStr
token_type: str = "bearer"
refresh_token: SecretStr = SecretStr("")
expires_in: int | None = Field(default=None, gt=0)
async def _token(config: OAuthConfig, body: str, transport: httpx.AsyncBaseTransport | None) -> TokenGrant:
client: Final = get_async_httpx_client(
llm_provider=httpxSpecialProvider.ROICalculator,
params={"timeout": 30, "follow_redirects": False, "transport": transport},
).client
try:
response: Final = await client.post(
config.token_url, data=dict(parse_qsl(body)), headers={"Accept": "application/json"}
)
except httpx.RequestError:
raise HTTPException(502, "Could not reach the provider. Try connecting again.") from None
finally:
if transport is not None:
await client.aclose()
if response.status_code != 200:
raise HTTPException(502, "The provider rejected the connection. Try connecting again.")
try:
result: Final = TokenGrant.model_validate(response.json())
except ValueError:
raise HTTPException(502, "The provider did not return a valid token. Try connecting again.") from None
if not result.access_token.get_secret_value() or result.token_type.casefold() != "bearer":
raise HTTPException(502, "The provider returned an unsupported token.")
return result
async def exchange_code(
config: OAuthConfig, state: OAuthState, code: str, transport: httpx.AsyncBaseTransport | None = None
) -> TokenGrant:
client: Final = WebApplicationClient(config.client_id)
body: Final = TypeAdapter(str).validate_python(
client.prepare_request_body( # pyright: ignore[reportUnknownMemberType] # oauthlib leaves extension kwargs untyped
code=code,
redirect_uri=config.redirect_uri,
code_verifier=state.verifier.get_secret_value(),
client_secret=config.client_secret.get_secret_value(),
)
)
return await _token(config, body, transport)
async def save_grant(
repository: ConfigRepository,
config: OAuthConfig,
grant: TokenGrant,
*,
revision: int | None = None,
previous: ROISettings | None = None,
attempt: int = 0,
) -> ROISettings:
stored: Final = await load_stored_settings(repository)
selected: Final = next(
(entry for entry in stored_connections(stored) if entry.id == connection_id(config.provider, config.api_url)),
None,
)
scoped: Final = select_connection(stored, selected) if selected else stored
current: Final = await load_settings(repository, scoped)
if revision is not None and stored.revision != revision:
raise HTTPException(409, "The connection changed during authorization. Start again from Connections.")
if previous is not None and (
current.source_provider,
current.source_api_url,
current.connection_type,
current.gitlab_token if current.source_provider == "gitlab" else current.github_token,
current.oauth_refresh_token,
) != (
previous.source_provider,
previous.source_api_url,
previous.connection_type,
previous.gitlab_token if previous.source_provider == "gitlab" else previous.github_token,
previous.oauth_refresh_token,
):
return current
changed: Final = (current.source_provider, current.source_api_url) != (config.provider, config.api_url)
refresh_token: Final = (
grant.refresh_token
if grant.refresh_token.get_secret_value() or previous is None
else previous.oauth_refresh_token
)
fields: Final[Mapping[str, object]] = {
"report_mode": "observed" if previous is None else current.report_mode,
"source_provider": config.provider,
"connection_type": "app",
"repos": () if changed else current.repos,
"identity_map": {} if changed else current.identity_map,
"ignored_logins": () if changed else current.ignored_logins,
"oauth_refresh_token": refresh_token,
"oauth_expires_at": datetime.now(timezone.utc) + timedelta(seconds=grant.expires_in)
if grant.expires_in
else None,
("github_token" if config.provider == "github" else "gitlab_token"): grant.access_token,
("github_api_url" if config.provider == "github" else "gitlab_api_url"): config.api_url,
}
settings: Final = ROISettings.model_validate({**current.model_dump(), **fields})
encrypted: Final = TypeAdapter(str).validate_python(encrypt_value_helper(grant.access_token.get_secret_value()))
try:
await save_settings(
repository,
settings,
encrypted if config.provider == "github" else scoped.github_token,
stored.estimator_key,
encrypted if config.provider == "gitlab" else scoped.gitlab_token,
revision=stored.revision,
)
return settings
except HTTPException as exc:
if exc.status_code != 409 or previous is None or attempt == 2:
raise
return await save_grant(repository, config, grant, revision=revision, previous=previous, attempt=attempt + 1)
def _expired(settings: ROISettings) -> bool:
return bool(
settings.connection_type == "app"
and settings.oauth_expires_at is not None
and settings.oauth_expires_at <= datetime.now(timezone.utc) + timedelta(minutes=5)
)
async def connected_settings(
repository: ConfigRepository, transport: httpx.AsyncBaseTransport | None = None, selected_id: str | None = None
) -> ROISettings:
initial: Final = await load_settings(repository, selected_id=selected_id)
if not _expired(initial):
return initial
selected: Final = selected_id or active_connection(await load_stored_settings(repository)).id
store: Final = SyncStore(repository.prisma_client, "roi_oauth_refresh_" + selected)
owner: Final = secrets.token_urlsafe(24)
status: Final = ROISyncStatus(
running=True,
phase="spend",
stage="Refreshing connection",
done=0,
total=0,
estimated=0,
reused=0,
needs_attention=0,
error=None,
)
async def wait_for_connection(attempt: int) -> ROISettings:
if attempt >= 100:
raise HTTPException(409, "The connection is refreshing. Try again shortly.")
settings: Final = await load_settings(repository, selected_id=selected)
if not _expired(settings):
return settings
if not await store.acquire(owner, status):
await asyncio.sleep(0.1)
return await wait_for_connection(attempt + 1)
try:
current: Final = await load_settings(repository, selected_id=selected)
if not _expired(current):
return current
config: Final = oauth_config(current.source_provider)
if (
config is None
or config.api_url != current.source_api_url
or not current.oauth_refresh_token.get_secret_value()
):
raise HTTPException(409, "The app connection expired. Reconnect from Connections.")
client: Final = WebApplicationClient(config.client_id)
body: Final = TypeAdapter(str).validate_python(
client.prepare_refresh_body( # pyright: ignore[reportUnknownMemberType] # oauthlib leaves extension kwargs untyped
refresh_token=current.oauth_refresh_token.get_secret_value(),
client_id=config.client_id,
client_secret=config.client_secret.get_secret_value(),
)
)
grant: Final = await _token(config, body, transport)
return await save_grant(repository, config, grant, previous=current)
finally:
await store.finish(owner, status.model_copy(update={"running": False, "phase": "complete"}))
return await wait_for_connection(0)

191
litellm/proxy/roi_calculator/observed_analytics.py Normal file
View file

@ -0,0 +1,191 @@
import calendar
import re
from collections.abc import Mapping
from datetime import date, datetime, timedelta, timezone
from itertools import chain
from statistics import median
from types import MappingProxyType
from typing import Final
from litellm.proxy.roi_calculator.analytics import normalize_email
from litellm.proxy.roi_calculator.branch_spend import attribute_branch_keys
from litellm.types.roi_observed import (
ObservedAccount,
ObservedData,
ObservedHumanSummary,
ObservedPeriod,
ObservedPeriodData,
ObservedPeriods,
ObservedPerson,
ObservedPersonPeriod,
ObservedPersonPeriods,
ObservedPull,
ObservedPullPeriods,
ObservedPullResponse,
ObservedReport,
ObservedWindow,
)
def reporting_windows(now: datetime, days: int = 28) -> tuple[ObservedWindow, ObservedWindow, ObservedWindow]:
end: Final = now.astimezone(timezone.utc).date() - timedelta(days=1)
start: Final = end - timedelta(days=days - 1)
last_year_end: Final = date(end.year - 1, end.month, min(end.day, calendar.monthrange(end.year - 1, end.month)[1]))
return (
ObservedWindow(start=start, end=end),
ObservedWindow(start=start - timedelta(days=days), end=start - timedelta(days=1)),
ObservedWindow(start=last_year_end - timedelta(days=days - 1), end=last_year_end),
)
def declared_requester(author: str, body: str) -> str:
if author.casefold().removesuffix("[bot]") not in ("devin-ai-integration", "devin-ai"):
return ""
matches: Final = frozenset(
match.group(1) for match in re.finditer(r"^Requested by:\s*@([A-Za-z0-9_.-]+)\s*$", body, re.MULTILINE)
)
return next(iter(matches)).casefold() if len(matches) == 1 else ""
def merge_hours(pull: ObservedPull) -> float | None:
if pull.created_at is None or pull.created_at.tzinfo is None or pull.merged_at.tzinfo is None:
return None
seconds: Final = (pull.merged_at - pull.created_at).total_seconds()
return seconds / 3600 if seconds >= 0 else None
def median_hours(pulls: tuple[ObservedPull, ...]) -> float | None:
values: Final = tuple(hours for pull in pulls if (hours := merge_hours(pull)) is not None)
return median(values) if values else None
def _owner_login(pull: ObservedPull) -> str:
login: Final = (pull.requester if pull.agent else pull.author).casefold()
return f"{pull.connection_id}:{login}" if pull.connection_id and login else login
def identity_matches(data: ObservedData, manual: Mapping[str, str], ignored: tuple[str, ...] = ()) -> Mapping[str, str]:
pulls: Final = tuple(chain(data.current.pulls, data.previous.pulls, data.last_year.pulls))
candidates: Final = frozenset((_owner_login(pull), normalize_email(pull.profile_email)) for pull in pulls)
gateway_emails: Final = frozenset(data.gateway_emails)
profiles: Final = {
login: email
for login, email in candidates
if login not in ignored
and email in gateway_emails
and len({value for key, value in candidates if key == login and value}) == 1
}
return MappingProxyType({**profiles, **manual})
def _person_period(data: ObservedPeriodData, email: str, identities: Mapping[str, str]) -> ObservedPersonPeriod:
pulls: Final = tuple(pull for pull in data.pulls if identities.get(_owner_login(pull)) == email)
spend: Final = tuple(row for row in data.spend if row["email"] == email)
cost: Final = sum(row["spend"] for row in spend)
days: Final = (data.window.end - data.window.start).days + 1
return ObservedPersonPeriod(
merged_prs=len(pulls),
prs_per_week=len(pulls) * 7 / days,
median_merge_hours=median_hours(pulls),
direct_authored=sum(not pull.agent for pull in pulls),
declared_agent_owned=sum(pull.agent for pull in pulls),
gateway_recorded_spend=cost,
recorded_spend_per_attributed_pr=cost / len(pulls) if spend and pulls else None,
spend_observation="records_present" if spend else "no_records",
pr_urls=tuple(pull.url for pull in pulls),
)
def _person(data: ObservedData, email: str, identities: Mapping[str, str]) -> ObservedPerson:
return ObservedPerson(
name=email.split("@", 1)[0],
email=email,
logins=tuple(
sorted(frozenset(login.rsplit(":", 1)[-1] for login, address in identities.items() if address == email))
),
accounts=tuple(
ObservedAccount(
connection_id=login.split(":", 1)[0] if ":" in login else "", login=login.rsplit(":", 1)[-1]
)
for login, address in identities.items()
if address == email
),
periods=ObservedPersonPeriods(
current=_person_period(data.current, email, identities),
previous=_person_period(data.previous, email, identities),
last_year=_person_period(data.last_year, email, identities),
),
)
def _issue_count(data: ObservedPeriodData, labels: frozenset[str]) -> int | None:
if data.issues is None:
return None
return sum(bool(labels.intersection(label.casefold().strip() for label in issue.labels)) for issue in data.issues)
def _period(data: ObservedPeriodData, identities: Mapping[str, str]) -> ObservedPeriod:
matched: Final = tuple(pull for pull in data.pulls if _owner_login(pull) in identities)
emails: Final = frozenset(identities.values())
spend: Final = tuple(row for row in data.spend if row["email"] in emails)
humans: Final = tuple(pull for pull in data.pulls if pull.author and not pull.agent)
return ObservedPeriod(
window=data.window,
merged_prs=len(data.pulls),
median_merge_hours=median_hours(data.pulls),
human_authored=len(humans),
agent_authored=sum(pull.agent for pull in data.pulls),
missing_author=sum(not pull.author for pull in data.pulls),
agents_without_requester=sum(pull.agent and not pull.requester for pull in data.pulls),
matched_internal_prs=len(matched),
new_bug_labeled_issues=_issue_count(data, frozenset(("bug", "kind:bug", "type::bug"))),
new_regression_labeled_issues=_issue_count(
data, frozenset(("regression", "kind:regression", "type::regression"))
),
explicitly_titled_revert_prs=sum(
bool(re.match(r"^revert(?:\W|$)", pull.title, re.IGNORECASE)) for pull in data.pulls
),
matched_users_recorded_spend=sum(row["spend"] for row in spend),
spend_observation="records_present" if spend else "no_records",
human_summary=ObservedHumanSummary(median_merge_hours=median_hours(humans)),
)
def _pulls(data: ObservedPeriodData) -> tuple[ObservedPullResponse, ...]:
costs: Final = attribute_branch_keys(
tuple((pull.url, pull.number, pull.source_repo, pull.source_branch) for pull in data.pulls), data.branch_spend
)
return tuple(
ObservedPullResponse.model_validate(
{**pull.model_dump(), "merge_hours": merge_hours(pull), "branch_cost": costs[(pull.url, pull.number)]}
)
for pull in data.pulls
)
def summarize_observed(data: ObservedData, manual: Mapping[str, str], ignored: tuple[str, ...] = ()) -> ObservedReport:
identities: Final = identity_matches(data, manual, ignored)
all_pulls: Final = tuple(chain(data.current.pulls, data.previous.pulls, data.last_year.pulls))
current_pulls: Final = _pulls(data.current)
linked_branches: Final = frozenset(
(pull.source_repo, pull.source_branch) for pull in current_pulls if pull.branch_cost.status == "matched"
)
return ObservedReport(
source_provider=data.source_provider,
connections=data.connections,
repos=data.repos,
captured_at=data.captured_at,
periods=ObservedPeriods(
current=_period(data.current, identities),
previous=_period(data.previous, identities),
last_year=_period(data.last_year, identities),
),
people=tuple(_person(data, email, identities) for email in sorted(frozenset(identities.values()))),
pulls=ObservedPullPeriods(
current=current_pulls, previous=_pulls(data.previous), last_year=_pulls(data.last_year)
),
unlinked_branches=tuple(
row for row in data.current.branch_spend or () if (row.repo, row.branch) not in linked_branches
),
unmatched_logins=tuple(sorted(frozenset(_owner_login(pull) for pull in all_pulls) - identities.keys() - {""})),
)

265
litellm/proxy/roi_calculator/observed_sync.py Normal file
View file

@ -0,0 +1,265 @@
import asyncio
from collections.abc import Awaitable, Callable, Mapping
from contextlib import suppress
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import Final, TypeAlias
from uuid import uuid4
import httpx
from litellm._logging import verbose_proxy_logger
from litellm.proxy.roi_calculator.analytics import normalize_email
from litellm.proxy.roi_calculator.github import GitHub, GitHubPullListItem, SourceError
from litellm.proxy.roi_calculator.github_observed import GitHubObserved
from litellm.proxy.roi_calculator.gitlab import GitLab
from litellm.proxy.roi_calculator.observed_analytics import declared_requester, reporting_windows
from litellm.proxy.roi_calculator.source import repository_tag
from litellm.proxy.roi_calculator.sync import BranchSpendReader, GatewayUserReader, SpendReader
from litellm.proxy.roi_calculator.sync_store import SyncStore
from litellm.types.roi_calculator import ROISettings, ROISyncStatus
from litellm.types.roi_observed import ObservedData, ObservedIssue, ObservedPeriodData, ObservedPull, ObservedWindow
def _author(pull: GitHubPullListItem) -> str:
return pull.user.login or "" if pull.user else ""
def _agent(pull: GitHubPullListItem) -> bool:
login: Final = _author(pull)
return (
bool(pull.user and pull.user.type == "Bot")
or login.endswith("[bot]")
or bool(login.startswith(("project_", "group_")) and "_bot_" in login)
)
def _owner(pull: GitHubPullListItem) -> str:
login: Final = _author(pull)
return declared_requester(login, pull.body or "") if _agent(pull) else login
def _public_email(pull: GitHubPullListItem) -> str:
return normalize_email(pull.user.email) if pull.user and not _agent(pull) else ""
def _pull(settings: ROISettings, repo: str, pull: GitHubPullListItem, profiles: Mapping[str, str]) -> ObservedPull:
if not pull.merged_at:
raise SourceError("The source returned an unmerged change. No partial report was saved.")
return ObservedPull(
repo=repo,
number=pull.number,
title=pull.title,
url=pull.html_url,
author=_author(pull),
agent=_agent(pull),
requester=declared_requester(_author(pull), pull.body or "") if _agent(pull) else "",
profile_email=profiles.get(_owner(pull).casefold(), "") or _public_email(pull),
created_at=pull.created_at,
merged_at=datetime.fromisoformat(pull.merged_at.replace("Z", "+00:00")),
source_repo=repository_tag(settings, pull.head.repo.full_name) if pull.head and pull.head.repo else "",
source_branch=pull.head.ref if pull.head else "",
)
async def collect_observed(
settings: ROISettings,
spend_reader: SpendReader,
gateway_user_reader: GatewayUserReader,
branch_spend_reader: BranchSpendReader,
now: datetime,
progress: Callable[[str, int, int], None],
transport: httpx.AsyncBaseTransport | None = None,
days: int = 28,
) -> ObservedData:
source: Final = GitLab(settings, transport) if settings.source_provider == "gitlab" else GitHub(settings, transport)
activity: Final = (
GitHubObserved(settings, source.client)
if settings.source_provider == "github" and settings.github_token.get_secret_value()
else source
)
windows: Final = reporting_windows(now.astimezone(timezone.utc), days)
slots: Final = asyncio.Semaphore(4)
total: Final = len(settings.repos) * 3
async def profile(login: str) -> tuple[str, str]:
async with slots:
return login.casefold(), await source.profile_email(login)
async def repository(
repo: str, window: ObservedWindow
) -> tuple[tuple[ObservedPull, ...], tuple[ObservedIssue, ...] | None]:
raw: Final = await activity.pulls(repo, window.start, window.end)
unique: Final = {(repo, item.number): item for item in raw}
if len(unique) != len(raw):
raise SourceError("The source returned duplicate changes. Retry to get a complete report.")
owners: Final = (
frozenset(_owner(pull) for pull in raw if not _public_email(pull))
- {""}
- settings.identity_map.keys()
- frozenset(settings.ignored_logins)
)
profiles: Final = MappingProxyType(dict(await asyncio.gather(*(profile(login) for login in owners))))
pulls: Final = tuple(_pull(settings, repo, item, profiles) for item in raw)
issues: Final = await activity.issues(repo, window.start, window.end)
return pulls, issues
async def period(window: ObservedWindow, offset: int) -> ObservedPeriodData:
async def read(index: int, repo: str) -> tuple[tuple[ObservedPull, ...], tuple[ObservedIssue, ...] | None]:
progress(f"Reading {repo} ({window.start} to {window.end})", offset + index, total)
return await repository(repo, window)
results: Final = tuple([await read(index, repo) for index, repo in enumerate(settings.repos)])
pulls: Final = tuple(chain.from_iterable(result[0] for result in results))
issues: Final = (
None
if results and all(result[1] is None for result in results)
else tuple(chain.from_iterable(result[1] or () for result in results))
)
branches: Final = tuple(
sorted(
frozenset(
(
*(repository_tag(settings, repo) for repo in settings.repos),
*(pull.source_repo for pull in pulls),
)
)
- {""}
)
)
return ObservedPeriodData(
window=window,
pulls=tuple(sorted(pulls, key=lambda pull: (pull.merged_at, pull.repo, pull.number), reverse=True)),
issues=issues,
spend=await spend_reader(window.start, window.end),
branch_spend=await branch_spend_reader(window.start, window.end, branches),
)
try:
gateway_emails: Final = await gateway_user_reader()
current: Final = await period(windows[0], 0)
previous: Final = await period(windows[1], len(settings.repos))
last_year: Final = await period(windows[2], len(settings.repos) * 2)
progress("Saving report", total, total)
return ObservedData(
source_provider=settings.source_provider,
source_api_url=settings.source_api_url,
repos=settings.repos,
captured_at=now,
gateway_emails=tuple(sorted(gateway_emails)),
current=current,
previous=previous,
last_year=last_year,
)
finally:
await source.close()
Progress: TypeAlias = Callable[[str, int, int], None]
BuildReport: TypeAlias = Callable[[Progress], Awaitable[ObservedData]]
class ObservedSyncManager:
def __init__(self) -> None:
self.status: ROISyncStatus = ROISyncStatus(
running=False,
phase="idle",
stage="Not synced",
done=0,
total=0,
estimated=0,
reused=0,
needs_attention=0,
error=None,
)
self._task: asyncio.Task[None] | None = None
self._lock: Final = asyncio.Lock()
def _progress(self, stage: str, done: int, total: int) -> None:
self.status = self.status.model_copy(
update={"phase": "repositories", "stage": stage, "done": done, "total": total}
)
async def start(self, build: BuildReport, store: SyncStore, scheduled_interval: float = 0) -> bool:
async with self._lock:
if self._task is not None and not self._task.done():
return False
status: Final = ROISyncStatus(
running=True,
phase="repositories",
stage="Reading repository activity",
done=0,
total=0,
estimated=0,
reused=0,
needs_attention=0,
error=None,
started_at=datetime.now(timezone.utc).isoformat(),
)
owner: Final = str(uuid4())
if not await store.acquire(owner, status, scheduled_interval):
return False
self.status = status
self._task = asyncio.create_task(self._run(build, store, owner))
return True
async def cancel(self) -> None:
if self._task is not None and not self._task.done():
self._task.cancel()
with suppress(asyncio.CancelledError):
await self._task
async def _heartbeat(self, store: SyncStore, owner: str, task: asyncio.Task[object] | None) -> None:
if task is None:
return
try:
while True:
await asyncio.sleep(5)
if not await store.heartbeat(owner, self.status):
task.cancel()
return
except Exception: # noqa: BLE001 # loss of the database lease must stop publication
task.cancel()
async def _run(self, build: BuildReport, store: SyncStore, owner: str) -> None:
monitor: Final = asyncio.create_task(self._heartbeat(store, owner, asyncio.current_task()))
try:
report: Final = await build(self._progress)
monitor.cancel()
with suppress(asyncio.CancelledError):
await monitor
complete: Final = self.status.model_copy(
update={
"running": False,
"phase": "complete",
"stage": "Up to date",
"finished_at": datetime.now(timezone.utc).isoformat(),
}
)
if not await store.finish(owner, complete, report):
raise SourceError("The sync was cancelled or replaced. The previous report was kept.")
self.status = complete
except asyncio.CancelledError:
self.status = self.status.model_copy(update={"phase": "cancelled", "stage": "Sync cancelled"})
raise
except SourceError as exc:
self.status = self.status.model_copy(update={"phase": "error", "stage": "Sync failed", "error": str(exc)})
except Exception: # noqa: BLE001 # background tasks must persist a safe error without exposing credentials
verbose_proxy_logger.exception("Observed ROI sync failed")
self.status = self.status.model_copy(
update={
"phase": "error",
"stage": "Sync failed",
"error": "Could not finish syncing. The previous report was kept. Retry after checking the connection.",
}
)
finally:
monitor.cancel()
with suppress(asyncio.CancelledError):
await monitor
self.status = self.status.model_copy(
update={"running": False, "finished_at": datetime.now(timezone.utc).isoformat()}
)
if self.status.phase != "complete":
await store.finish(owner, self.status)

135
litellm/proxy/roi_calculator/observed_workspace.py Normal file
View file

@ -0,0 +1,135 @@
from collections.abc import Mapping
from datetime import date, datetime
from itertools import chain
from types import MappingProxyType
from typing import Final
import httpx
from litellm.proxy.roi_calculator.github import SourceError
from litellm.proxy.roi_calculator.observed_analytics import reporting_windows, summarize_observed
from litellm.proxy.roi_calculator.observed_sync import Progress, collect_observed
from litellm.proxy.roi_calculator.settings import StoredConnection, connection_id
from litellm.proxy.roi_calculator.source import repository_tag
from litellm.proxy.roi_calculator.sync import BranchSpendReader, GatewayUserReader, SpendReader
from litellm.types.roi_calculator import ROISettings, ROISpendRecord
from litellm.types.roi_observed import ObservedData, ObservedPeriodData, ObservedReport, ObservedSource
def source_details(settings: ROISettings) -> ObservedSource:
return ObservedSource(
id=connection_id(settings.source_provider, settings.source_api_url),
source_provider=settings.source_provider,
api_url=settings.source_api_url,
repos=settings.repos,
)
def scoped_data(data: ObservedData, source: ObservedSource) -> ObservedData:
def period(value: ObservedPeriodData) -> ObservedPeriodData:
return value.model_copy(
update={"pulls": tuple(pull.model_copy(update={"connection_id": source.id}) for pull in value.pulls)}
)
return data.model_copy(
update={
"connections": (source,),
"current": period(data.current),
"previous": period(data.previous),
"last_year": period(data.last_year),
}
)
def summarize_workspace(data: ObservedData, connections: tuple[StoredConnection, ...]) -> ObservedReport:
included: Final = frozenset(source.id for source in data.connections)
active: Final = tuple(entry for entry in connections if entry.id in included)
maps: Final = ({f"{entry.id}:{login}": email for login, email in entry.identity_map.items()} for entry in active)
identities: Final = MappingProxyType(dict(chain.from_iterable(mapping.items() for mapping in maps)))
ignored: Final = tuple(
chain.from_iterable(tuple(f"{entry.id}:{login}" for login in entry.ignored_logins) for entry in connections)
)
return summarize_observed(data, identities, ignored)
def combine_observed(sources: tuple[ObservedData, ...], repos: tuple[str, ...]) -> ObservedData:
first: Final = sources[0]
def period(values: tuple[ObservedPeriodData, ...]) -> ObservedPeriodData:
if any(value.window != values[0].window or value.spend != values[0].spend for value in values):
raise SourceError("The reporting windows changed during sync. Retry to get a complete report.")
pulls: Final = tuple(chain.from_iterable(value.pulls for value in values))
if len({pull.url for pull in pulls}) != len(pulls):
raise SourceError("A repository is selected through more than one connection. Select it once.")
return ObservedPeriodData(
window=values[0].window,
pulls=tuple(sorted(pulls, key=lambda pull: (pull.merged_at, pull.url), reverse=True)),
issues=None
if all(value.issues is None for value in values)
else tuple(chain.from_iterable(value.issues or () for value in values)),
spend=values[0].spend,
branch_spend=None
if any(value.branch_spend is None for value in values)
else tuple(
{
(row.repo, row.branch): row
for row in chain.from_iterable(value.branch_spend or () for value in values)
}.values()
),
)
providers: Final = frozenset(source.source_provider for source in sources)
return ObservedData(
source_provider=first.source_provider if len(providers) == 1 else "mixed",
source_api_url=first.source_api_url if len(sources) == 1 else "",
connections=tuple(chain.from_iterable(source.connections for source in sources)),
repos=repos,
captured_at=first.captured_at,
gateway_emails=first.gateway_emails,
current=period(tuple(source.current for source in sources)),
previous=period(tuple(source.previous for source in sources)),
last_year=period(tuple(source.last_year for source in sources)),
)
async def collect_workspace(
connections: tuple[tuple[ROISettings, BranchSpendReader], ...],
spend_reader: SpendReader,
gateway_user_reader: GatewayUserReader,
now: datetime,
progress: Progress,
transport: httpx.AsyncBaseTransport | None = None,
days: int = 28,
) -> ObservedData:
windows: Final = reporting_windows(now, days)
spending: Final[Mapping[tuple[date, date], tuple[ROISpendRecord, ...]]] = {
(window.start, window.end): await spend_reader(window.start, window.end) for window in windows
}
emails: Final = await gateway_user_reader()
total: Final = sum(len(settings.repos) * 3 for settings, _reader in connections)
async def spend(start: date, end: date) -> tuple[ROISpendRecord, ...]:
return spending[(start, end)]
async def users() -> frozenset[str]:
return emails
async def collect(index: int, settings: ROISettings, branch_reader: BranchSpendReader) -> ObservedData:
offset: Final = sum(len(prior.repos) * 3 for prior, _reader in connections[:index])
def update(stage: str, done: int, _total: int) -> None:
progress(stage, offset + done, total)
data: Final = await collect_observed(settings, spend, users, branch_reader, now, update, transport, days=days)
return scoped_data(data, source_details(settings))
data: Final = tuple(
[await collect(index, settings, reader) for index, (settings, reader) in enumerate(connections)]
)
repos: Final = tuple(
chain.from_iterable(
tuple(repository_tag(settings, repo) if len(connections) > 1 else repo for repo in settings.repos)
for settings, _reader in connections
)
)
return combine_observed(data, repos)

253
litellm/proxy/roi_calculator/settings.py Normal file
View file

@ -0,0 +1,253 @@
from collections.abc import Mapping
from datetime import datetime
from hashlib import sha256
from types import MappingProxyType
from typing import Annotated, Final, Literal
from fastapi import Depends, HTTPException
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper
from litellm.repositories.config_repository import ConfigRepository
from litellm.types.roi_calculator import DEFAULT_PROMPT, ROISettings
_SETTINGS_KEY: Final = "roi_calculator_settings"
def connection_id(provider: str, api_url: str) -> str:
return provider + "_" + sha256(api_url.strip().rstrip("/").encode()).hexdigest()[:16]
class StoredConnection(BaseModel):
model_config = ConfigDict(frozen=True)
source_provider: Literal["github", "gitlab"]
api_url: str
token: str = ""
connection_type: Literal["token", "app"] = "token"
oauth_refresh_token: str = ""
oauth_expires_at: datetime | None = None
repos: tuple[str, ...] = ()
identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({}))
ignored_logins: tuple[str, ...] = ()
@property
def id(self) -> str:
return connection_id(self.source_provider, self.api_url)
class StoredROISettings(BaseModel):
model_config = ConfigDict(extra="ignore")
revision: int = 0
report_mode: Literal["legacy", "observed"] = "legacy"
source_provider: Literal["github", "gitlab"] = "github"
connection_type: Literal["token", "app"] = "token"
oauth_refresh_token: str = ""
oauth_expires_at: datetime | None = None
ignored_logins: tuple[str, ...] = ()
gitlab_api_url: str = "https://gitlab.com/api/v4"
gitlab_token: str = ""
github_api_url: str = "https://api.github.com"
github_token: str = ""
estimator_key: str = ""
repos: tuple[str, ...] = ()
estimator_model: str = ""
estimator_prompt: str = DEFAULT_PROMPT
backfill_days: int = Field(default=7, ge=1, le=3650)
update_interval_minutes: float = Field(default=1440, ge=0, le=43200)
identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({}))
connections: tuple[StoredConnection, ...] = ()
def active_connection(stored: StoredROISettings) -> StoredConnection:
return StoredConnection(
source_provider=stored.source_provider,
api_url=stored.gitlab_api_url if stored.source_provider == "gitlab" else stored.github_api_url,
token=stored.gitlab_token if stored.source_provider == "gitlab" else stored.github_token,
connection_type=stored.connection_type,
oauth_refresh_token=stored.oauth_refresh_token,
oauth_expires_at=stored.oauth_expires_at,
repos=stored.repos,
identity_map=stored.identity_map,
ignored_logins=stored.ignored_logins,
)
def stored_connections(stored: StoredROISettings) -> tuple[StoredConnection, ...]:
active: Final = active_connection(stored)
if not stored.connections and not active.repos and not active.token:
return ()
return tuple({**{entry.id: entry for entry in stored.connections}, active.id: active}.values())
def select_connection(stored: StoredROISettings, selected: StoredConnection) -> StoredROISettings:
return stored.model_copy(
update={
"source_provider": selected.source_provider,
"connection_type": selected.connection_type,
"oauth_refresh_token": selected.oauth_refresh_token,
"oauth_expires_at": selected.oauth_expires_at,
"repos": selected.repos,
"identity_map": selected.identity_map,
"ignored_logins": selected.ignored_logins,
("gitlab_api_url" if selected.source_provider == "gitlab" else "github_api_url"): selected.api_url,
("gitlab_token" if selected.source_provider == "gitlab" else "github_token"): selected.token,
}
)
async def read_admin(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> UserAPIKeyAuth:
if user_api_key_dict.user_role not in (
LitellmUserRoles.PROXY_ADMIN,
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
):
raise HTTPException(status_code=403, detail="Only proxy admins can access the ROI Calculator.")
return user_api_key_dict
async def write_admin(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> UserAPIKeyAuth:
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="Only proxy admins can change ROI Calculator settings.")
return user_api_key_dict
async def get_roi_config_repository(
_user: Annotated[UserAPIKeyAuth, Depends(read_admin)],
) -> ConfigRepository:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail=CommonProxyErrors.db_not_connected_error.value,
)
return ConfigRepository(prisma_client, use_writer=True)
async def load_stored_settings(repository: ConfigRepository, selected_id: str | None = None) -> StoredROISettings:
parameter: Final = await repository.get_param(_SETTINGS_KEY)
if parameter is None:
if selected_id is not None:
raise HTTPException(404, "This connection no longer exists. Reload Connections.")
return StoredROISettings()
try:
stored: Final = StoredROISettings.model_validate(parameter.param_value)
except ValidationError:
raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None
if selected_id is None:
return stored
selected: Final = next((entry for entry in stored_connections(stored) if entry.id == selected_id), None)
if selected is None:
raise HTTPException(404, "This connection no longer exists. Reload Connections.")
return select_connection(stored, selected)
async def load_settings(
repository: ConfigRepository, value: StoredROISettings | None = None, selected_id: str | None = None
) -> ROISettings:
stored: Final = value if value is not None else await load_stored_settings(repository, selected_id)
token: Final = decrypt_value_helper(stored.github_token, _SETTINGS_KEY) if stored.github_token else ""
try:
return ROISettings(
report_mode=stored.report_mode,
source_provider=stored.source_provider,
connection_type=stored.connection_type,
oauth_refresh_token=SecretStr(decrypt_value_helper(stored.oauth_refresh_token, _SETTINGS_KEY) or "")
if stored.oauth_refresh_token
else SecretStr(""),
oauth_expires_at=stored.oauth_expires_at,
ignored_logins=stored.ignored_logins,
gitlab_api_url=stored.gitlab_api_url,
gitlab_token=SecretStr(decrypt_value_helper(stored.gitlab_token, _SETTINGS_KEY) or "")
if stored.gitlab_token
else SecretStr(""),
github_api_url=stored.github_api_url,
github_token=SecretStr(token or ""),
estimator_key=SecretStr(decrypt_value_helper(stored.estimator_key, _SETTINGS_KEY) or "")
if stored.estimator_key
else SecretStr(""),
update_interval_minutes=stored.update_interval_minutes,
repos=stored.repos,
estimator_model=stored.estimator_model,
estimator_prompt=stored.estimator_prompt,
backfill_days=stored.backfill_days,
identity_map=stored.identity_map,
)
except ValidationError:
raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None
async def save_settings(
repository: ConfigRepository,
settings: ROISettings,
encrypted_token: str,
encrypted_estimator_key: str,
encrypted_gitlab_token: str = "",
revision: int = 0,
replace_connection_id: str | None = None,
) -> None:
previous: Final = await load_stored_settings(repository)
stored: Final = StoredROISettings(
revision=revision + 1,
report_mode=settings.report_mode,
source_provider=settings.source_provider,
connection_type=settings.connection_type,
oauth_refresh_token=TypeAdapter(str).validate_python(
encrypt_value_helper(settings.oauth_refresh_token.get_secret_value())
)
if settings.oauth_refresh_token.get_secret_value()
else "",
oauth_expires_at=settings.oauth_expires_at,
ignored_logins=settings.ignored_logins,
gitlab_api_url=settings.gitlab_api_url,
gitlab_token=encrypted_gitlab_token,
github_api_url=settings.github_api_url,
github_token=encrypted_token,
estimator_key=encrypted_estimator_key,
update_interval_minutes=settings.update_interval_minutes,
repos=settings.repos,
estimator_model=settings.estimator_model,
estimator_prompt=settings.estimator_prompt,
backfill_days=settings.backfill_days,
identity_map=settings.identity_map,
)
active: Final = active_connection(stored)
combined: Final = stored.model_copy(
update={
"connections": tuple(
{
**{entry.id: entry for entry in stored_connections(previous) if entry.id != replace_connection_id},
active.id: active,
}.values()
)
}
)
if not await repository.set_param_if_revision(_SETTINGS_KEY, combined.model_dump(mode="json"), revision):
raise HTTPException(409, "Settings changed while you were editing. Reload and try again.")
async def enable_observed_reporting(repository: ConfigRepository) -> None:
stored: Final = await load_stored_settings(repository)
if stored.report_mode == "observed":
return
updated: Final = stored.model_copy(update={"report_mode": "observed", "revision": stored.revision + 1})
if not await repository.set_param_if_revision(_SETTINGS_KEY, updated.model_dump(mode="json"), stored.revision):
raise HTTPException(409, "Settings changed while starting the report. Reload and try again.")
async def save_connection_identities(
repository: ConfigRepository, stored: StoredROISettings, connections: tuple[StoredConnection, ...]
) -> None:
active: Final = next(entry for entry in connections if entry.id == active_connection(stored).id)
updated: Final = select_connection(stored, active).model_copy(
update={"connections": connections, "revision": stored.revision + 1}
)
if not await repository.set_param_if_revision(_SETTINGS_KEY, updated.model_dump(mode="json"), stored.revision):
raise HTTPException(409, "Settings changed while you were editing. Reload and try again.")

37
litellm/proxy/roi_calculator/sync_store.py
View file

@ -1,11 +1,12 @@
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final, Protocol, cast # noqa: TID251 - PrismaWrapper dynamically delegates database methods
from typing import Final, Literal, Protocol, cast # noqa: TID251 - PrismaWrapper dynamically delegates database methods
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.proxy.utils import PrismaClient
from litellm.types.roi_calculator import ROIReport, ROISyncStatus
from litellm.types.roi_observed import ObservedData
_SYNC_KEY: Final = "roi_calculator_sync"
_REPORT_KEY: Final = "roi_calculator_report"
@ -30,8 +31,14 @@ class _SyncDatabase(Protocol):
class SyncStore:
def __init__(self, prisma: PrismaClient) -> None:
def __init__(
self,
prisma: PrismaClient,
namespace: Literal["roi_calculator", "roi_observed", "roi_oauth_refresh"] = "roi_calculator",
) -> None:
self._db: Final = cast(_SyncDatabase, prisma.writer_db) # cast-ok: PrismaWrapper delegates methods dynamically
self._sync_key: Final = namespace + "_sync"
self._report_key: Final = namespace + "_report"
async def acquire(self, owner: str, status: ROISyncStatus, scheduled_interval: float = 0) -> bool:
rows: Final = await self._db.query_raw(
@ -43,7 +50,7 @@ class SyncStore:
OR "LiteLLM_Config".param_value->'status'->>'running' = 'false')
AND ($3::text::double precision = 0 OR "LiteLLM_Config".last_run_at <= NOW() - $3::text::double precision * INTERVAL '1 minute')
RETURNING param_name""",
_SYNC_KEY,
self._sync_key,
_SyncState(owner=owner, status=status).model_dump_json(),
str(scheduled_interval),
)
@ -58,14 +65,20 @@ class SyncStore:
AND param_value->'status'->>'running' = 'true'
AND last_run_at >= NOW() - INTERVAL '60 seconds'
RETURNING param_name""",
_SYNC_KEY,
self._sync_key,
owner,
status.model_dump_json(),
)
return bool(rows)
async def finish(self, owner: str, status: ROISyncStatus, report: ROIReport | None = None) -> bool:
report_json: Final = TypeAdapter(ROIReport).dump_json(report).decode() if report is not None else None
async def finish(self, owner: str, status: ROISyncStatus, report: ROIReport | ObservedData | None = None) -> bool:
report_json: Final = (
report.model_dump_json()
if isinstance(report, ObservedData)
else TypeAdapter(ROIReport).dump_json(report).decode()
if report is not None
else None
)
rows: Final = await self._db.query_raw(
"""WITH owned AS (
SELECT param_name FROM "LiteLLM_Config"
@ -92,11 +105,11 @@ class SyncStore:
UPDATE "LiteLLM_Config" SET param_value = jsonb_set(param_value, '{status}', $3::jsonb),
last_run_at = NOW()
WHERE param_name IN (SELECT param_name FROM owned) RETURNING param_name""",
_SYNC_KEY,
self._sync_key,
owner,
status.model_dump_json(),
report_json,
_REPORT_KEY,
self._report_key,
)
return bool(rows)
@ -105,7 +118,7 @@ class SyncStore:
await self._db.query_raw(
"""SELECT param_value, last_run_at, last_run_at < NOW() - INTERVAL '60 seconds' AS expired
FROM "LiteLLM_Config" WHERE param_name = $1""",
_SYNC_KEY,
self._sync_key,
)
)
if not rows:
@ -119,7 +132,7 @@ class SyncStore:
"phase": "error",
"finished_at": rows[0].last_run_at.replace(tzinfo=timezone.utc).isoformat(),
"stage": "Sync interrupted",
"error": "The worker stopped responding. Run analysis again to resume saved estimates.",
"error": "The worker stopped responding. Sync again to refresh the report.",
}
)
)
@ -136,8 +149,8 @@ class SyncStore:
)
), last_run_at = NOW()
WHERE param_name = $1 AND param_value->'status'->>'running' = 'true' """,
_SYNC_KEY,
self._sync_key,
)
async def clear_report(self) -> None:
await self._db.execute_raw('DELETE FROM "LiteLLM_Config" WHERE param_name = $1', _REPORT_KEY)
await self._db.execute_raw('DELETE FROM "LiteLLM_Config" WHERE param_name = $1', self._report_key)

24
litellm/repositories/config_repository.py
View file

@ -33,6 +33,10 @@ class _ConfigTable(Protocol):
async def delete(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ...
class _ConfigDatabase(Protocol):
async def query_raw(self, query: str, *args: object) -> object: ...
class ConfigParam:
"""Simple wrapper for config parameter from DB."""
@ -85,6 +89,26 @@ class ConfigRepository:
)
return ConfigParam(param_name=param_name, param_value=param_value)
async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool:
delegate: Final = self.prisma_client.writer_db
database: Final = cast(_ConfigDatabase, delegate) # cast-ok: Prisma delegates database methods dynamically
rows: Final = await database.query_raw(
"""INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at)
SELECT $1, $2::jsonb, NOW() WHERE $3::int = 0
ON CONFLICT (param_name) DO UPDATE
SET param_value = EXCLUDED.param_value, last_run_at = NOW()
WHERE COALESCE(("LiteLLM_Config".param_value->>'revision')::int, 0) = $3::int
RETURNING param_name"""
if revision == 0
else """UPDATE "LiteLLM_Config" SET param_value = $2::jsonb, last_run_at = NOW()
WHERE param_name = $1 AND COALESCE((param_value->>'revision')::int, 0) = $3::int
RETURNING param_name""",
param_name,
json.dumps(param_value),
revision,
)
return bool(rows)
async def delete_param(self, param_name: str) -> bool:
"""Delete a config parameter from the database."""
try:

15
litellm/types/roi_calculator.py
View file

@ -1,4 +1,5 @@
from collections.abc import Mapping
from datetime import datetime
from types import MappingProxyType
from typing import Final, Literal
@ -24,7 +25,12 @@ def normalize_source_login(value: str, provider: str = "github") -> str:
class ROISettings(BaseModel):
model_config = ConfigDict(frozen=True)
report_mode: Literal["legacy", "observed"] = "legacy"
source_provider: Literal["github", "gitlab"] = "github"
connection_type: Literal["token", "app"] = "token"
oauth_refresh_token: SecretStr = SecretStr("")
oauth_expires_at: datetime | None = None
ignored_logins: tuple[str, ...] = ()
gitlab_api_url: str = "https://gitlab.com/api/v4"
gitlab_token: SecretStr = SecretStr("")
github_api_url: str = "https://api.github.com"
@ -74,8 +80,13 @@ class ROISettings(BaseModel):
import re
normalized_values: Final = tuple(repo.strip().rstrip("/").removesuffix(".git") for repo in values)
repository_keys: Final = tuple(
repo.casefold() if info.data.get("source_provider") != "gitlab" else repo for repo in normalized_values
)
normalized: Final = tuple(
repo for index, repo in enumerate(normalized_values) if repo not in normalized_values[:index]
repo
for index, repo in enumerate(normalized_values)
if repository_keys[index] not in repository_keys[:index]
)
pattern: Final = (
r"[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+"
@ -121,6 +132,7 @@ class ROISettings(BaseModel):
class ROISettingsUpdate(BaseModel):
model_config = ConfigDict(extra="forbid")
report_mode: Literal["legacy", "observed"] | None = None
source_provider: Literal["github", "gitlab"] | None = None
gitlab_api_url: str | None = None
gitlab_token: str | None = None
@ -140,6 +152,7 @@ class ROIEstimatorModel(BaseModel):
class ROISettingsResponse(BaseModel):
report_mode: Literal["legacy", "observed"] = "legacy"
source_provider: Literal["github", "gitlab"] = "github"
gitlab_api_url: str = "https://gitlab.com/api/v4"
has_gitlab_token: bool = False

213
litellm/types/roi_observed.py Normal file
View file

@ -0,0 +1,213 @@
from collections.abc import Mapping
from datetime import date, datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from litellm.types.roi_calculator import ROIBranchAttribution, ROIBranchSpend, ROISpendRecord
class ObservedModel(BaseModel):
model_config = ConfigDict(frozen=True)
class ObservedWindow(ObservedModel):
start: date
end: date
class ObservedSource(ObservedModel):
id: str
source_provider: Literal["github", "gitlab"]
api_url: str
repos: tuple[str, ...]
class ObservedAccount(ObservedModel):
connection_id: str
login: str
class ObservedPull(ObservedModel):
connection_id: str = ""
repo: str
number: int
title: str
url: str
author: str
agent: bool = False
requester: str = ""
profile_email: str = ""
created_at: datetime | None = None
merged_at: datetime
source_repo: str = ""
source_branch: str = ""
class ObservedIssue(ObservedModel):
repo: str
number: int
created_at: datetime
labels: tuple[str, ...] = ()
class ObservedPeriodData(ObservedModel):
window: ObservedWindow
pulls: tuple[ObservedPull, ...]
issues: tuple[ObservedIssue, ...] | None
spend: tuple[ROISpendRecord, ...]
branch_spend: tuple[ROIBranchSpend, ...] | None = None
class ObservedData(ObservedModel):
source_provider: Literal["github", "gitlab", "mixed"]
connections: tuple[ObservedSource, ...] = ()
source_api_url: str
repos: tuple[str, ...]
captured_at: datetime
gateway_emails: tuple[str, ...]
current: ObservedPeriodData
previous: ObservedPeriodData
last_year: ObservedPeriodData
class ObservedPersonPeriod(ObservedModel):
merged_prs: int
prs_per_week: float
median_merge_hours: float | None
direct_authored: int
declared_agent_owned: int
gateway_recorded_spend: float
recorded_spend_per_attributed_pr: float | None
spend_observation: Literal["records_present", "no_records"]
pr_urls: tuple[str, ...]
class ObservedPersonPeriods(ObservedModel):
current: ObservedPersonPeriod
previous: ObservedPersonPeriod
last_year: ObservedPersonPeriod
class ObservedPerson(ObservedModel):
name: str
email: str
logins: tuple[str, ...]
accounts: tuple[ObservedAccount, ...] = ()
periods: ObservedPersonPeriods
class ObservedHumanSummary(ObservedModel):
median_merge_hours: float | None
class ObservedPeriod(ObservedModel):
window: ObservedWindow
merged_prs: int
median_merge_hours: float | None
human_authored: int
agent_authored: int
missing_author: int
agents_without_requester: int
matched_internal_prs: int
new_bug_labeled_issues: int | None
new_regression_labeled_issues: int | None
explicitly_titled_revert_prs: int
matched_users_recorded_spend: float
spend_observation: Literal["records_present", "no_records"]
human_summary: ObservedHumanSummary
class ObservedPeriods(ObservedModel):
current: ObservedPeriod
previous: ObservedPeriod
last_year: ObservedPeriod
class ObservedPullResponse(ObservedPull):
merge_hours: float | None
branch_cost: ROIBranchAttribution
class ObservedPullPeriods(ObservedModel):
current: tuple[ObservedPullResponse, ...]
previous: tuple[ObservedPullResponse, ...]
last_year: tuple[ObservedPullResponse, ...]
class ObservedReport(ObservedModel):
source_provider: Literal["github", "gitlab", "mixed"]
connections: tuple[ObservedSource, ...] = ()
repos: tuple[str, ...]
captured_at: datetime
periods: ObservedPeriods
people: tuple[ObservedPerson, ...]
pulls: ObservedPullPeriods
unlinked_branches: tuple[ROIBranchSpend, ...]
unmatched_logins: tuple[str, ...]
class ObservedReportResponse(ObservedModel):
report: ObservedReport | None
class ObservedIdentityUpdate(BaseModel):
model_config = ConfigDict(extra="forbid")
email: str
logins: tuple[str, ...] = Field(default=(), max_length=100)
accounts: tuple[ObservedAccount, ...] | None = Field(default=None, max_length=500)
class ObservedConnectionIdentities(ObservedSource):
identity_map: Mapping[str, str]
unmatched_logins: tuple[str, ...]
class ObservedIdentities(ObservedModel):
gateway_emails: tuple[str, ...]
identity_map: Mapping[str, str]
unmatched_logins: tuple[str, ...]
connections: tuple[ObservedConnectionIdentities, ...] = ()
class ObservedConnection(ObservedModel):
id: str = ""
source_provider: Literal["github", "gitlab"]
api_url: str
repos: tuple[str, ...]
has_token: bool
update_interval_minutes: float
ready: bool
connection_type: Literal["token", "app"]
class ObservedSettings(ObservedConnection):
connections: tuple[ObservedConnection, ...] = ()
class ObservedSettingsUpdate(BaseModel):
model_config = ConfigDict(extra="forbid")
connection_id: str | None = Field(default=None, max_length=100)
source_provider: Literal["github", "gitlab"]
api_url: str
token: str | None = None
repos: tuple[str, ...]
update_interval_minutes: float | None = Field(default=None, ge=0, le=43200, allow_inf_nan=False)
class ObservedApp(ObservedModel):
configured: bool
can_install: bool = False
api_url: str | None = None
callback_url: str | None = None
class ObservedApps(ObservedModel):
github: ObservedApp
gitlab: ObservedApp
class ObservedAuthorization(ObservedModel):
url: str

4
tests/integration/README.md
View file

@ -2,6 +2,8 @@
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
The ROI database contracts in `database/test_roi_observed.py` run in the GitHub Actions `roi-database` Postgres shard and upload coverage on each PR. They own temporary databases and script only the external provider transport. `GITHUB_FILES` in `run.py` assigns these files to GitHub Actions and excludes them from the CircleCI selection
The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist; the `mcp` job does this in CI, so MCP tests must own their resources per scenario. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
@ -12,7 +14,7 @@ The generated lifecycle models use 20 examples, eight steps, generation and shri
Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change
The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. Existing GitHub Actions jobs do not own these tests
The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. GitHub Actions runs only the explicit `GITHUB_FILES` set in `run.py`
There is no per-node manifest. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration/<directory>/test_*.py` file in a scheduled group as owned by CircleCI

6
tests/integration/conftest.py
View file

@ -16,6 +16,7 @@ from tests.integration._support.client import Gateway, eventually, gateway_from_
from tests.integration._support.generation import LIFECYCLE_SETTINGS
from tests.integration._support.manifest import OWNED_DIRECTORIES
from tests.integration._support.routing import RoutingPlugin
from tests.integration.run import GITHUB_FILES
COLLECTED: Final = pytest.StashKey[tuple[str, ...]]()
REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]()
@ -69,7 +70,10 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item
for item in items
if item.path.is_relative_to(root) and item.path.relative_to(root).parts[0] in OWNED_DIRECTORIES
)
if owned and os.environ.get("GITHUB_ACTIONS") == "true":
circleci_only: Final = tuple(
item for item in owned if item.path.relative_to(root.parents[1]).as_posix() not in GITHUB_FILES
)
if circleci_only and os.environ.get("GITHUB_ACTIONS") == "true":
raise pytest.UsageError("Integration contracts are owned by CircleCI")
for item in owned:
item.add_marker(pytest.mark.integration)

823
tests/integration/database/test_roi_observed.py Normal file
View file

@ -0,0 +1,823 @@
import asyncio
from collections.abc import AsyncIterator
from datetime import date, datetime, timezone
from typing import Final
from urllib.parse import parse_qs, urlsplit
import httpx
import pytest
import pytest_asyncio
from fastapi import FastAPI, HTTPException
from pydantic import SecretStr
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.roi_calculator.github import SourceError
from litellm.proxy.roi_calculator.oauth import (
OAuthConfig,
TokenGrant,
begin_authorization,
connected_settings,
consume_state,
exchange_code,
save_grant,
)
from litellm.proxy.roi_calculator.observed_sync import ObservedSyncManager, Progress
from litellm.proxy.roi_calculator.settings import load_settings, load_stored_settings, save_settings
from litellm.proxy.roi_calculator.sync_store import SyncStore
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.config_repository import ConfigRepository
from litellm.types.roi_observed import ObservedData, ObservedPeriodData, ObservedWindow
from tests.integration._support.database import scratch_database, write_rows
@pytest_asyncio.fixture(loop_scope="function")
async def repository(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[ConfigRepository]:
with scratch_database() as url:
write_rows(
'CREATE TABLE "LiteLLM_Config" (param_name TEXT PRIMARY KEY, param_value JSONB NOT NULL, '
"last_run_at TIMESTAMP NOT NULL DEFAULT NOW(), reload_revision BIGINT NOT NULL DEFAULT 0)",
(),
database_url=url,
)
monkeypatch.setenv("DATABASE_URL", url)
monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False)
monkeypatch.setenv("LITELLM_SALT_KEY", "test-only-observed-roi-salt-0123456789")
client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache()))
await client.connect()
try:
yield ConfigRepository(client, use_writer=True)
finally:
await client.disconnect()
def _config(provider: str = "github") -> OAuthConfig:
from pydantic import TypeAdapter
from litellm.proxy.roi_calculator.oauth import Provider
selected: Final = TypeAdapter(Provider).validate_python(provider)
return OAuthConfig(
selected,
"https://api.github.com" if provider == "github" else "https://gitlab.com/api/v4",
f"https://{provider}.com",
"test-client",
SecretStr("test-client-secret"),
"https://gateway.example.test",
)
def _report() -> ObservedData:
period: Final = ObservedPeriodData(
window=ObservedWindow(start=date(2026, 9, 1), end=date(2026, 9, 28)), pulls=(), issues=(), spend=()
)
return ObservedData(
source_provider="github",
source_api_url="https://api.github.com",
repos=("org/repo",),
captured_at=datetime.now(timezone.utc),
gateway_emails=(),
current=period,
previous=period,
last_year=period,
)
@pytest.mark.asyncio
async def test_settings_compare_and_swap_rejects_stale_writers(repository: ConfigRepository) -> None:
assert await repository.set_param_if_revision("settings", {"revision": 1, "account": "ari"}, 0)
assert not await repository.set_param_if_revision("settings", {"revision": 1, "account": "bea"}, 0)
writes: Final = await asyncio.gather(
*(repository.set_param_if_revision("settings", {"revision": 2, "account": name}, 1) for name in ("bea", "cam"))
)
assert sum(writes) == 1
saved: Final = await repository.get_param("settings")
assert saved is not None and saved.param_value in (
{"revision": 2, "account": "bea"},
{"revision": 2, "account": "cam"},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("provider", ("github", "gitlab"))
async def test_authorization_uses_pkce_single_use_state_and_encrypted_credentials(
repository: ConfigRepository, provider: str
) -> None:
config: Final = _config(provider)
url, nonce = await begin_authorization(repository, config)
params: Final = parse_qs(urlsplit(url).query)
assert params["code_challenge_method"] == ["S256"]
assert params["redirect_uri"] == [config.redirect_uri]
state: Final = await consume_state(repository, params["state"][0], nonce, config)
assert state.verifier.get_secret_value() != "**********"
def respond(request: httpx.Request) -> httpx.Response:
body: Final = parse_qs(request.content.decode())
assert body["code_verifier"] == [state.verifier.get_secret_value()]
assert body["code"] == ["test-code"]
assert body["client_secret"] == ["test-client-secret"]
assert body["redirect_uri"] == [config.redirect_uri]
return httpx.Response(
200, json={"access_token": "test-access", "refresh_token": "test-refresh", "expires_in": 3600}
)
grant: Final = await exchange_code(config, state, "test-code", httpx.MockTransport(respond))
await save_grant(repository, config, grant, revision=state.settings_revision)
stored: Final = await load_stored_settings(repository)
assert "test-access" not in stored.model_dump_json() and "test-refresh" not in stored.model_dump_json()
connected: Final = await load_settings(repository)
assert (
connected.github_token if provider == "github" else connected.gitlab_token
).get_secret_value() == "test-access"
assert connected.oauth_refresh_token.get_secret_value() == "test-refresh"
with pytest.raises(HTTPException, match="already used"):
await consume_state(repository, params["state"][0], nonce, config)
@pytest.mark.asyncio
async def test_authorization_rejects_a_different_browser_and_changed_settings(repository: ConfigRepository) -> None:
config: Final = _config()
url, nonce = await begin_authorization(repository, config)
state: Final = parse_qs(urlsplit(url).query)["state"][0]
with pytest.raises(HTTPException, match="same browser"):
await consume_state(repository, state, "another-browser", config)
new_url, new_nonce = await begin_authorization(repository, config)
verified: Final = await consume_state(repository, parse_qs(urlsplit(new_url).query)["state"][0], new_nonce, config)
await save_grant(repository, config, TokenGrant(access_token=SecretStr("first")))
with pytest.raises(HTTPException, match="changed during authorization"):
await save_grant(
repository, config, TokenGrant(access_token=SecretStr("stale")), revision=verified.settings_revision
)
assert (await load_settings(repository)).github_token.get_secret_value() == "first"
assert nonce != new_nonce
@pytest.mark.asyncio
async def test_concurrent_app_refresh_rotates_once_and_preserves_account_links(
repository: ConfigRepository, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_ID", "test-client")
monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_SECRET", "test-client-secret")
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.test")
await save_grant(
repository,
_config(),
TokenGrant(access_token=SecretStr("old-access"), refresh_token=SecretStr("old-refresh"), expires_in=1),
)
stored: Final = await load_stored_settings(repository)
settings: Final = (await load_settings(repository)).model_copy(update={"identity_map": {"ari": "ari@example.test"}})
await save_settings(repository, settings, stored.github_token, stored.estimator_key, revision=stored.revision)
requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue()
def respond(request: httpx.Request) -> httpx.Response:
requests.put_nowait(request)
assert parse_qs(request.content.decode())["refresh_token"] == ["old-refresh"]
return httpx.Response(
200, json={"access_token": "new-access", "refresh_token": "new-refresh", "expires_in": 3600}
)
results: Final = await asyncio.gather(
*(connected_settings(repository, httpx.MockTransport(respond)) for _ in range(12))
)
assert requests.qsize() == 1
assert all(result.github_token.get_secret_value() == "new-access" for result in results)
assert all(result.identity_map == {"ari": "ari@example.test"} for result in results)
assert (await load_settings(repository)).oauth_refresh_token.get_secret_value() == "new-refresh"
@pytest.mark.asyncio
async def test_refresh_cannot_restore_a_connection_replaced_while_the_provider_responds(
repository: ConfigRepository, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_ID", "test-client")
monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_SECRET", "test-client-secret")
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.test")
await save_grant(
repository,
_config(),
TokenGrant(access_token=SecretStr("old"), refresh_token=SecretStr("refresh"), expires_in=1),
)
started: Final = asyncio.Event()
release: Final = asyncio.Event()
async def respond(request: httpx.Request) -> httpx.Response:
started.set()
await release.wait()
return httpx.Response(200, json={"access_token": "late-token", "expires_in": 3600})
pending: Final = asyncio.create_task(connected_settings(repository, httpx.MockTransport(respond)))
await asyncio.wait_for(started.wait(), 2)
await save_grant(repository, _config(), TokenGrant(access_token=SecretStr("replacement"), expires_in=3600))
release.set()
result: Final = await asyncio.wait_for(pending, 2)
assert result.github_token.get_secret_value() == "replacement"
assert (await load_settings(repository)).github_token.get_secret_value() == "replacement"
@pytest.mark.asyncio
async def test_failed_cancelled_and_stale_workers_cannot_replace_the_published_report(
repository: ConfigRepository,
) -> None:
store: Final = SyncStore(repository.prisma_client, "roi_observed")
manager: Final = ObservedSyncManager()
report: Final = _report()
await repository.set_param("roi_observed_report", report.model_dump(mode="json"))
release: Final = asyncio.Event()
async def build(progress: Progress) -> ObservedData:
await release.wait()
raise SourceError("source failure")
assert await manager.start(build, store)
assert not await ObservedSyncManager().start(build, store)
await store.cancel()
release.set()
await manager.cancel()
published: Final = await repository.get_param("roi_observed_report")
assert published is not None and ObservedData.model_validate(published.param_value) == report
async def complete(progress: Progress) -> ObservedData:
return report
assert await manager.start(complete, store)
async def finished() -> None:
for _ in range(200):
status: Final = await store.status()
if status is not None and not status.running:
assert status.phase == "complete", status
return
await asyncio.sleep(0.01)
pytest.fail("Observed report did not publish")
await asyncio.wait_for(finished(), 5)
saved: Final = await repository.get_param("roi_observed_report")
assert saved is not None and ObservedData.model_validate(saved.param_value) == report
@pytest.mark.asyncio
@pytest.mark.parametrize("provider,install_first", (("github", False), ("github", True), ("gitlab", False)))
async def test_app_callback_round_trip_and_admin_authorization(
repository: ConfigRepository, monkeypatch: pytest.MonkeyPatch, provider: str, install_first: bool
) -> None:
from fastapi import FastAPI
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.roi_observed_endpoints import (
get_oauth_repository,
get_observed_transport,
router,
)
from litellm.proxy.roi_calculator.settings import get_roi_config_repository
config: Final = _config(provider)
monkeypatch.setenv(f"LITELLM_ROI_{provider.upper()}_CLIENT_ID", config.client_id)
monkeypatch.setenv(f"LITELLM_ROI_{provider.upper()}_CLIENT_SECRET", config.client_secret.get_secret_value())
monkeypatch.setenv("PROXY_BASE_URL", config.proxy_url)
monkeypatch.setenv("LITELLM_ROI_GITHUB_APP_SLUG", "example-roi")
def respond(request: httpx.Request) -> httpx.Response:
if request.method == "POST":
assert str(request.url) == config.token_url
if parse_qs(request.content.decode()).get("code") == ["rejected-code"]:
return httpx.Response(400, json={"error": "invalid_grant"})
return httpx.Response(
200,
json={
"access_token": "provider-test-access",
"refresh_token": "provider-test-refresh",
"expires_in": 3600,
},
)
assert request.headers["Authorization"] == "Bearer provider-test-access"
assert "PRIVATE-TOKEN" not in request.headers
return httpx.Response(200, json=[])
app: Final = FastAPI()
app.include_router(router)
app.dependency_overrides[get_roi_config_repository] = lambda: repository
app.dependency_overrides[get_oauth_repository] = lambda: repository
app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=config.proxy_url) as client:
for role, expected in (
(LitellmUserRoles.INTERNAL_USER, 403),
(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, 403),
(LitellmUserRoles.PROXY_ADMIN, 200),
):
app.dependency_overrides[user_api_key_auth] = lambda role=role: UserAPIKeyAuth(user_role=role)
response: Final = await client.post(f"/roi-calculator/observed/oauth/{provider}/start")
assert response.status_code == expected, response.text
successful: Final = await client.post(
f"/roi-calculator/observed/oauth/{provider}/start", params={"install": str(install_first).lower()}
)
assert "HttpOnly" in successful.headers["set-cookie"] and "Secure" in successful.headers["set-cookie"]
initial_state: Final = parse_qs(urlsplit(successful.json()["url"]).query)["state"][0]
installed: Final = (
await client.get("/roi-calculator/observed/oauth/github/installed", params={"state": initial_state})
if install_first
else None
)
if install_first:
assert urlsplit(successful.json()["url"]).path == "/apps/example-roi/installations/new"
assert installed is not None
assert installed.status_code == 303, installed.text
assert urlsplit(installed.headers["location"]).path == "/login/oauth/authorize"
assert parse_qs(urlsplit(installed.headers["location"]).query)["state"][0] != initial_state
state: Final = (
parse_qs(urlsplit(installed.headers["location"]).query)["state"][0] if installed else initial_state
)
callback: Final = await client.get(
f"/roi-calculator/observed/oauth/{provider}/callback", params={"state": state, "code": "test-code"}
)
assert callback.status_code == 303, callback.text
assert callback.headers["location"] == config.proxy_url + f"/ui/roi-calculator/?connected={provider}"
saved: Final = await client.get("/roi-calculator/observed/settings")
assert saved.json()["has_token"] is True and saved.json()["connection_type"] == "app"
assert "provider-test-access" not in saved.text and "provider-test-refresh" not in saved.text
replay: Final = await client.get(
f"/roi-calculator/observed/oauth/{provider}/callback", params={"state": state, "code": "test-code"}
)
assert replay.status_code == 303
assert replay.headers["location"] == config.proxy_url + "/ui/roi-calculator/?connection_failed=1"
restart: Final = await client.post(f"/roi-calculator/observed/oauth/{provider}/start")
denied_state: Final = parse_qs(urlsplit(restart.json()["url"]).query)["state"][0]
denied: Final = await client.get(
f"/roi-calculator/observed/oauth/{provider}/callback",
params={"state": denied_state, "error": "access_denied"},
)
assert denied.status_code == 303
assert denied.headers["location"] == config.proxy_url + "/ui/roi-calculator/?connection_cancelled=1"
unchanged: Final = await client.get("/roi-calculator/observed/settings")
assert unchanged.json() == saved.json()
expired_installation: Final = (
await client.get("/roi-calculator/observed/oauth/github/installed", params={"state": initial_state})
if install_first
else None
)
if expired_installation is not None:
assert expired_installation.status_code == 303
assert expired_installation.headers["location"].endswith("?connection_failed=1")
retry: Final = await client.post(f"/roi-calculator/observed/oauth/{provider}/start")
rejected: Final = await client.get(
f"/roi-calculator/observed/oauth/{provider}/callback",
params={"state": parse_qs(urlsplit(retry.json()["url"]).query)["state"][0], "code": "rejected-code"},
)
assert rejected.status_code == 303
assert rejected.headers["location"] == config.proxy_url + "/ui/roi-calculator/?connection_failed=1"
assert "litellm_roi_oauth" not in client.cookies
assert (await client.get("/roi-calculator/observed/settings")).json() == saved.json()
@pytest.mark.asyncio
async def test_identity_api_combines_accounts_and_removes_automatic_links_without_resync(
repository: ConfigRepository,
) -> None:
from fastapi import FastAPI
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.roi_observed_endpoints import router
from litellm.proxy.roi_calculator.settings import get_roi_config_repository
from litellm.types.roi_observed import ObservedPull
write_rows('CREATE TABLE "LiteLLM_UserTable" (user_id TEXT PRIMARY KEY, user_email TEXT)', ())
write_rows(
'INSERT INTO "LiteLLM_UserTable" VALUES (%s,%s),(%s,%s)', ("ari", "ari@example.test", "bea", "bea@example.test")
)
settings: Final = (await load_settings(repository)).model_copy(update={"repos": ("org/repo",)})
await save_settings(repository, settings, "", "")
base: Final = _report()
pulls: Final = tuple(
ObservedPull(
repo="org/repo",
number=index,
title="Change",
url=f"https://github.com/org/repo/pull/{index}",
author=login,
profile_email="ari@example.test" if login == "ari" else "",
merged_at=base.captured_at,
)
for index, login in enumerate(("ari", "old-ari"))
)
data: Final = base.model_copy(
update={
"gateway_emails": ("ari@example.test", "bea@example.test"),
"current": base.current.model_copy(update={"pulls": pulls}),
}
)
await repository.set_param("roi_observed_report", data.model_dump(mode="json"))
app: Final = FastAPI()
app.include_router(router)
app.dependency_overrides[get_roi_config_repository] = lambda: repository
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="https://gateway.example.test"
) as client:
linked: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "ari@example.test", "logins": ["ari", "old-ari"]}
)
assert linked.status_code == 200, linked.text
person: Final = linked.json()["report"]["people"][0]
assert person["logins"] == ["ari", "old-ari"] and person["periods"]["current"]["merged_prs"] == 2
conflict: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "bea@example.test", "logins": ["old-ari"]}
)
assert conflict.status_code == 409
unknown: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "missing@example.test", "logins": []}
)
assert unknown.status_code == 422
removed: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "ari@example.test", "logins": []}
)
assert removed.json()["report"]["people"] == []
assert [login.split(":", 1)[-1] for login in removed.json()["report"]["unmatched_logins"]] == ["ari", "old-ari"]
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
)
readonly: Final = await client.get("/roi-calculator/observed/report")
assert readonly.status_code == 200
denied: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "ari@example.test", "logins": []}
)
assert denied.status_code == 403
@pytest.mark.asyncio
async def test_both_app_connections_keep_repositories_credentials_and_account_links(
repository: ConfigRepository,
) -> None:
from litellm.proxy.roi_calculator.settings import connection_id, stored_connections
await save_grant(
repository,
_config(),
TokenGrant(access_token=SecretStr("github-access"), refresh_token=SecretStr("github-refresh")),
)
stored: Final = await load_stored_settings(repository)
github: Final = (await load_settings(repository)).model_copy(
update={"repos": ("org/service", "org/docs"), "identity_map": {"ari": "ari@example.test"}}
)
await save_settings(repository, github, stored.github_token, stored.estimator_key, revision=stored.revision)
await save_grant(
repository,
_config("gitlab"),
TokenGrant(access_token=SecretStr("gitlab-access"), refresh_token=SecretStr("gitlab-refresh")),
)
gitlab_id: Final = connection_id("gitlab", _config("gitlab").api_url)
github_id: Final = connection_id("github", _config().api_url)
first: Final = await load_settings(repository, selected_id=github_id)
second: Final = await load_settings(repository, selected_id=gitlab_id)
assert first.repos == ("org/service", "org/docs")
assert first.identity_map == {"ari": "ari@example.test"}
assert first.github_token.get_secret_value() == "github-access"
assert first.oauth_refresh_token.get_secret_value() == "github-refresh"
assert second.gitlab_token.get_secret_value() == "gitlab-access"
assert second.oauth_refresh_token.get_secret_value() == "gitlab-refresh"
assert second.identity_map == {} and second.repos == ()
await save_grant(repository, _config(), TokenGrant(access_token=SecretStr("github-rotated")), previous=first)
preserved: Final = await load_settings(repository, selected_id=gitlab_id)
assert preserved.model_dump(exclude={"github_token", "github_api_url"}) == second.model_dump(
exclude={"github_token", "github_api_url"}
)
assert (await load_settings(repository, selected_id=github_id)).github_token.get_secret_value() == "github-rotated"
persisted: Final = await load_stored_settings(repository)
assert len(stored_connections(persisted)) == 2
assert all(
token not in persisted.model_dump_json()
for token in ("github-access", "github-rotated", "gitlab-access", "github-refresh", "gitlab-refresh")
)
@pytest.mark.asyncio
async def test_cross_provider_account_form_saves_atomically_without_legacy_logins(repository: ConfigRepository) -> None:
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.roi_observed_endpoints import router
from litellm.proxy.roi_calculator.settings import get_roi_config_repository, stored_connections, write_admin
write_rows('CREATE TABLE "LiteLLM_UserTable" (user_id TEXT PRIMARY KEY, user_email TEXT)', ())
write_rows(
'INSERT INTO "LiteLLM_UserTable" VALUES (%s, %s), (%s, %s)',
("ari", "ari@example.test", "bea", "bea@example.test"),
)
await save_grant(repository, _config("github"), TokenGrant(access_token=SecretStr("github-token")))
await save_grant(repository, _config("gitlab"), TokenGrant(access_token=SecretStr("gitlab-token")))
connections: Final = stored_connections(await load_stored_settings(repository))
accounts: Final = tuple({"connection_id": entry.id, "login": "ari"} for entry in connections)
app: Final = FastAPI()
app.include_router(router)
async def config_repository() -> ConfigRepository:
return repository
async def admin() -> UserAPIKeyAuth:
return UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
app.dependency_overrides[get_roi_config_repository] = config_repository
app.dependency_overrides[write_admin] = admin
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client:
saved: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "ari@example.test", "accounts": accounts}
)
assert saved.status_code == 200, saved.text
after: Final = stored_connections(await load_stored_settings(repository))
assert all(entry.identity_map == {"ari": "ari@example.test"} for entry in after)
conflicting: Final = ({"connection_id": connections[0].id, "login": "bea"}, accounts[1])
rejected: Final = await client.put(
"/roi-calculator/observed/identities", json={"email": "bea@example.test", "accounts": conflicting}
)
assert rejected.status_code == 409, rejected.text
assert stored_connections(await load_stored_settings(repository)) == after
@pytest.mark.asyncio
async def test_http_sync_combines_providers_retains_period_and_recovers_invalid_reports(
repository: ConfigRepository,
) -> None:
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.roi_observed_endpoints import (
get_observed_manager,
get_observed_transport,
router,
)
from litellm.proxy.roi_calculator.settings import get_roi_config_repository
from litellm.types.roi_observed import ObservedReportResponse, ObservedSettings
write_rows('CREATE TABLE "LiteLLM_UserTable" (user_id TEXT PRIMARY KEY, user_email TEXT)', ())
write_rows(
'CREATE TABLE "LiteLLM_DailyUserSpend" (user_id TEXT, date TEXT, spend DOUBLE PRECISION, api_requests INTEGER)',
(),
)
write_rows(
'CREATE TABLE "LiteLLM_SpendLogs" (spend DOUBLE PRECISION, request_tags JSONB, metadata JSONB, "startTime" TIMESTAMP)',
(),
)
def respond(request: httpx.Request) -> httpx.Response:
if request.headers.get("Authorization") == "Bearer rejected":
return httpx.Response(401, json={"message": "Bad credentials"})
if request.url.path == "/graphql":
return httpx.Response(
200, json={"data": {"search": {"issueCount": 0, "nodes": [], "pageInfo": {"hasNextPage": False}}}}
)
if request.url.path.startswith("/repos/"):
return httpx.Response(200, json={"full_name": "org/service", "has_issues": True})
if request.url.path in ("/user/repos", "/api/v4/projects"):
return httpx.Response(200, json=[])
if request.url.path == "/api/v4/projects/org/service":
return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/service"})
if request.url.path.endswith(("/merge_requests", "/issues")):
return httpx.Response(200, json=[])
raise AssertionError(str(request.url))
manager: Final = ObservedSyncManager()
app: Final = FastAPI()
app.include_router(router)
app.dependency_overrides[get_roi_config_repository] = lambda: repository
app.dependency_overrides[get_observed_manager] = lambda: manager
app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
async def finish() -> None:
for _ in range(200):
status: Final = await SyncStore(repository.prisma_client, "roi_observed").status()
if status and not status.running:
assert status.phase == "complete", status.error
return
await asyncio.sleep(0.01)
pytest.fail("Sync did not finish")
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client:
assert (await client.get("/roi-calculator/observed/report")).json() == {"report": None}
assert (await client.post("/roi-calculator/observed/sync")).status_code == 409
for provider, url in (("github", "https://api.github.com"), ("gitlab", "https://gitlab.com/api/v4")):
result: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"source_provider": provider,
"api_url": url,
"token": "test-token",
"repos": ["org/service"],
"update_interval_minutes": 0,
},
)
assert result.status_code == 200, result.text
settings: Final = ObservedSettings.model_validate(
(await client.get("/roi-calculator/observed/settings")).json()
)
assert len(settings.connections) == 2 and all(entry.has_token for entry in settings.connections)
for entry in settings.connections:
repositories: Final = await client.get(
"/roi-calculator/observed/repositories", params={"connection": entry.id}
)
assert repositories.status_code == 200, repositories.text
rejected: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"source_provider": "github",
"api_url": "https://api.github.com",
"token": "rejected",
"repos": ["org/service"],
},
)
assert rejected.status_code == 502, rejected.text
assert (
ObservedSettings.model_validate((await client.get("/roi-calculator/observed/settings")).json()) == settings
)
for params in ({"days": 7}, {}):
started: Final = await client.post("/roi-calculator/observed/sync", params=params)
assert started.status_code == 202, started.text
await asyncio.wait_for(finish(), 5)
response: Final = await client.get("/roi-calculator/observed/report")
report: Final = ObservedReportResponse.model_validate(response.json()).report
assert report is not None and report.source_provider == "mixed"
assert len(report.connections) == 2 and report.periods.current.merged_prs == 0
assert all(
(period.window.end - period.window.start).days == 6
for period in (report.periods.current, report.periods.previous, report.periods.last_year)
)
await repository.set_param("roi_observed_report", {"invalid": True})
assert (await client.get("/roi-calculator/observed/report")).status_code == 500
recovered: Final = await client.post("/roi-calculator/observed/sync")
assert recovered.status_code == 202, recovered.text
await asyncio.wait_for(finish(), 5)
rebuilt: Final = ObservedReportResponse.model_validate(
(await client.get("/roi-calculator/observed/report")).json()
).report
assert (
rebuilt is not None
and (rebuilt.periods.current.window.end - rebuilt.periods.current.window.start).days == 27
)
@pytest.mark.asyncio
@pytest.mark.parametrize("provider", ("github", "gitlab"))
async def test_host_whitespace_preserves_app_credentials_and_account_matches(
repository: ConfigRepository, provider: str
) -> None:
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.roi_observed_endpoints import get_observed_transport, router
from litellm.proxy.roi_calculator.settings import connection_id, get_roi_config_repository
config: Final = _config(provider)
granted: Final = await save_grant(
repository,
config,
TokenGrant(access_token=SecretStr("test-access"), refresh_token=SecretStr("test-refresh"), expires_in=3600),
)
stored: Final = await load_stored_settings(repository)
before: Final = granted.model_copy(
update={"repos": ("org/service",), "identity_map": {"ari": "ari@example.test"}, "ignored_logins": ("bea",)}
)
await save_settings(
repository, before, stored.github_token, stored.estimator_key, stored.gitlab_token, revision=stored.revision
)
def respond(request: httpx.Request) -> httpx.Response:
assert request.headers.get("Authorization") == "Bearer test-access"
if "/repos/" in request.url.path:
return httpx.Response(200, json={"full_name": "org/service"})
if request.url.path.endswith("/merge_requests"):
return httpx.Response(200, json=[])
return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/service"})
app: Final = FastAPI()
app.include_router(router)
app.dependency_overrides[get_roi_config_repository] = lambda: repository
app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client:
response: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"connection_id": connection_id(provider, config.api_url),
"source_provider": provider,
"api_url": f" {config.api_url}/ ",
"repos": ["org/service"],
},
)
assert response.status_code == 200, response.text
assert (await load_settings(repository)) == before
@pytest.mark.asyncio
async def test_connection_edits_replace_only_the_selected_host_and_keep_the_workspace_schedule(
repository: ConfigRepository,
) -> None:
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.roi_observed_endpoints import get_observed_transport, router
from litellm.proxy.roi_calculator.settings import get_roi_config_repository, stored_connections
from litellm.types.roi_calculator import ROISettings
from litellm.types.roi_observed import ObservedSettings
legacy: Final = ROISettings(repos=("org/service",), estimator_model="legacy-model", update_interval_minutes=60)
await save_settings(repository, legacy, "", "")
def respond(request: httpx.Request) -> httpx.Response:
if "/repos/" in request.url.path:
return httpx.Response(200, json={"full_name": "org/service"})
if "/projects/" in request.url.path:
return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/service"})
raise AssertionError(str(request.url))
app: Final = FastAPI()
app.include_router(router)
app.dependency_overrides[get_roi_config_repository] = lambda: repository
app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client:
assert (await client.get("/roi-calculator/observed/settings")).status_code == 200
assert (await load_settings(repository)) == legacy
github: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"source_provider": "github",
"api_url": "https://api.github.com",
"token": "github-token",
"repos": ["org/service"],
"update_interval_minutes": 30,
},
)
assert github.status_code == 200, github.text
gitlab: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"source_provider": "gitlab",
"api_url": "https://gitlab.com/api/v4",
"token": "gitlab-token",
"repos": ["org/service"],
},
)
assert gitlab.status_code == 200, gitlab.text
before: Final = stored_connections(await load_stored_settings(repository))
edited: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"connection_id": github.json()["id"],
"source_provider": "github",
"api_url": "https://git.example.test/api/v3",
"token": "enterprise-token",
"repos": ["org/service"],
},
)
assert edited.status_code == 200, edited.text
settings: Final = ObservedSettings.model_validate(edited.json())
assert len(settings.connections) == 2
assert {entry.api_url for entry in settings.connections} == {
"https://git.example.test/api/v3",
"https://gitlab.com/api/v4",
}
assert all(entry.update_interval_minutes == 30 for entry in settings.connections)
after: Final = stored_connections(await load_stored_settings(repository))
assert next(entry for entry in after if entry.source_provider == "gitlab") == next(
entry for entry in before if entry.source_provider == "gitlab"
)
assert (await load_settings(repository)).report_mode == "observed"
stale: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"connection_id": github.json()["id"],
"source_provider": "github",
"api_url": "https://api.github.com",
"repos": [],
},
)
assert stale.status_code == 404, stale.text
duplicate: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"connection_id": settings.id,
"source_provider": "gitlab",
"api_url": "https://gitlab.com/api/v4",
"repos": [],
},
)
assert duplicate.status_code == 409, duplicate.text
assert stored_connections(await load_stored_settings(repository)) == after
manual: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"source_provider": "gitlab",
"api_url": "https://gitlab.com/api/v4",
"repos": ["org/service"],
"update_interval_minutes": 0,
},
)
assert manual.status_code == 200, manual.text
reselected: Final = await client.put(
"/roi-calculator/observed/settings",
json={
"connection_id": settings.id,
"source_provider": "github",
"api_url": "https://git.example.test/api/v3",
"repos": ["org/service"],
},
)
assert reselected.status_code == 200, reselected.text
assert all(
entry.update_interval_minutes == 0
for entry in ObservedSettings.model_validate(reselected.json()).connections
)

2
tests/integration/run.py
View file

@ -23,6 +23,7 @@ GROUPS: Final = MappingProxyType(
"security": ("security",),
}
)
GITHUB_FILES: Final = frozenset({"tests/integration/database/test_roi_observed.py"})
@dataclass(frozen=True, slots=True)
@ -63,6 +64,7 @@ def main() -> int:
str(path.relative_to(root))
for folder in GROUPS[options.group]
for path in sorted((root / "tests/integration" / folder).rglob("test_*.py"))
if str(path.relative_to(root)) not in GITHUB_FILES
)
if options.list:
print("\n".join(group_files))

90
tests/integration/spend/test_roi_branch_spend.py
View file

@ -1,6 +1,7 @@
import json
import os
import uuid
from collections.abc import Mapping
from datetime import date
from typing import Final
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@ -11,6 +12,8 @@ from prisma import Prisma
from psycopg import sql
from litellm.proxy.roi_calculator.branch_spend import read_branch_spend
from litellm.types.roi_calculator import ROIBranchSpend
from tests.integration._support.client import Gateway, JsonValue, object_value
@pytest.mark.asyncio
@ -46,7 +49,7 @@ async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() ->
setup.execute(
sql.SQL(
'INSERT INTO {}."LiteLLM_SpendLogs" ("startTime", spend, request_tags) '
'VALUES (%s::timestamp, %s, %s::jsonb)'
"VALUES (%s::timestamp, %s, %s::jsonb)"
).format(sql.Identifier(schema)),
(timestamp, spend, json.dumps(request_tags)),
)
@ -57,9 +60,9 @@ async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() ->
(None, 100, ("litellm-roi-estimator",)),
):
setup.execute(
sql.SQL('INSERT INTO {}."LiteLLM_SpendLogs" VALUES (%s::timestamp, %s, %s::jsonb, %s::jsonb)').format(
sql.Identifier(schema)
),
sql.SQL(
'INSERT INTO {}."LiteLLM_SpendLogs" VALUES (%s::timestamp, %s, %s::jsonb, %s::jsonb)'
).format(sql.Identifier(schema)),
(
"2026-09-15 00:00:00",
spend,
@ -77,3 +80,82 @@ async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() ->
assert costs == {"feature/one": (18, 3), "Feature/one": (7, 1), "free": (0, 1)}
finally:
setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
def test_documented_header_and_body_tags_reach_recorded_branch_and_pr_cost(gateway: Gateway) -> None:
import asyncio
from datetime import datetime, timezone
from litellm.proxy.roi_calculator.branch_spend import attribute_branch_keys
from tests.integration._support.client import eventually
from tests.integration._support.database import read_rows
from tests.integration._support.wire import Reply, Request, wire_server
marker: Final = uuid.uuid4().hex
repo: Final = f"github.com/integration/{marker}"
branch: Final = "feature/tag-attribution"
tags: Final = [f"repo:{repo}", f"branch:{branch}"]
def respond(request: Request) -> Reply:
body: Final = object_value(json.loads(request.body))
assert "tags" not in body and "x-litellm-tags" not in request.headers
return Reply(
body=json.dumps(
{
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1,
"model": "owned-model",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
}
).encode()
)
with wire_server(respond) as upstream, gateway.scenario() as scenario:
model: Final = scenario.model(
api_base=upstream.url + "/v1", input_cost_per_token=0.001, output_cost_per_token=0.002
)
examples: Final[tuple[tuple[Mapping[str, JsonValue], Mapping[str, str]], ...]] = (
({"metadata": {"tags": tags}}, {}),
({"tags": tags}, {}),
({}, {"x-litellm-tags": ", ".join(tags + tags)}),
)
for payload, headers in examples:
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": "tag attribution"}],
**payload,
},
headers=headers,
)
assert response.status_code == 200, response.text
rows: Final = eventually(
lambda: read_rows(
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_tags @> %s::jsonb', (json.dumps(tags),)
),
lambda values: len(values) == 3,
seconds=70,
)
expected: Final = 3 * (5 * 0.001 + 3 * 0.002)
assert sum(float(row["spend"]) for row in rows) == pytest.approx(expected)
async def recorded() -> tuple[ROIBranchSpend, ...]:
database: Final = Prisma()
await database.connect()
try:
today: Final = datetime.now(timezone.utc).date()
return await read_branch_spend(database, today, today, (repo,), casefold_repo=True)
finally:
await database.disconnect()
spending: Final = asyncio.run(recorded())
costs: Final = attribute_branch_keys(((repo, 1, repo, branch),), spending)
assert costs[(repo, 1)].spend == pytest.approx(expected)
assert costs[(repo, 1)].requests == 3
assert costs[(repo, 1)].status == "matched"

22
tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
View file

@ -120,6 +120,15 @@ class _ConfigRepository:
self.values = MappingProxyType({**self.values, param_name: param_value})
return self.values[param_name]
async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool:
from litellm.proxy.roi_calculator.settings import StoredROISettings
stored: Final = StoredROISettings.model_validate(self.values.get(param_name, {}))
if stored.revision != revision:
return False
await self.set_param(param_name, param_value)
return True
def _client(
role: LitellmUserRoles, repository: _ConfigRepository, transport: httpx.AsyncBaseTransport | None = None
@ -322,8 +331,14 @@ def test_schedule_rejects_intervals_under_five_minutes(interval: float) -> None:
@pytest.mark.parametrize("anchor", ("2026-09-30T12:00:00", "2026-09-30T12:00:00Z", "2026-09-30T14:00:00+02:00"))
def test_schedule_normalizes_legacy_and_offset_timestamps(anchor: str) -> None:
settings: Final = ROISettings(repos=("example/repo",), estimator_model="estimator", update_interval_minutes=60)
@pytest.mark.parametrize("observed", (False, True))
def test_schedule_normalizes_timestamps_and_respects_report_mode(anchor: str, observed: bool) -> None:
settings: Final = ROISettings(
repos=("example/repo",),
estimator_model="estimator",
update_interval_minutes=60,
report_mode="observed" if observed else "legacy",
)
status: Final = ROISyncStatus(
running=False,
phase="error",
@ -337,7 +352,8 @@ def test_schedule_normalizes_legacy_and_offset_timestamps(anchor: str) -> None:
finished_at=anchor,
)
report: Final = sample_report(datetime(2026, 9, 30, tzinfo=timezone.utc))
assert _next_update(settings, status, report) == datetime(2026, 9, 30, 13, tzinfo=timezone.utc)
expected: Final = None if observed else datetime(2026, 9, 30, 13, tzinfo=timezone.utc)
assert _next_update(settings, status, report) == expected
def test_manual_match_recalculates_saved_report_and_removal_restores_cohort() -> None:

148
tests/unit/proxy/roi_calculator/test_github_observed.py Normal file
View file

@ -0,0 +1,148 @@
import asyncio
import re
from datetime import date, datetime, timedelta, timezone
from typing import Final
from unittest.mock import AsyncMock
import httpx
import pytest
from pydantic import BaseModel, SecretStr
from litellm.proxy.roi_calculator.github import SourceError
from litellm.proxy.roi_calculator.github_observed import GitHubObserved
from litellm.types.roi_calculator import ROISettings
class _Variables(BaseModel):
q: str
after: str | None
class _Query(BaseModel):
variables: _Variables
def _node(number: int, merged: datetime) -> dict[str, object]:
return {
"number": number,
"url": f"https://github.com/org/repo/pull/{number}",
"title": "Change",
"createdAt": (merged - timedelta(seconds=16)).isoformat(),
"updatedAt": merged.isoformat(),
"mergedAt": merged.isoformat(),
"author": {"login": "ari", "__typename": "User"},
}
def _page(nodes: tuple[dict[str, object], ...], count: int, cursor: str | None = None) -> httpx.Response:
return httpx.Response(
200,
json={
"data": {
"search": {
"issueCount": count,
"nodes": nodes,
"pageInfo": {"hasNextPage": cursor is not None, "endCursor": cursor},
}
}
},
)
@pytest.mark.asyncio
async def test_large_history_splits_the_search_limit_without_losing_midnight_or_split_boundaries() -> None:
start: Final = datetime(2026, 9, 1, tzinfo=timezone.utc)
timestamps: Final = tuple(start + timedelta(seconds=index * 60) for index in range(1001))
def respond(request: httpx.Request) -> httpx.Response:
assert request.headers["Authorization"] == "Bearer test-only-token"
assert request.url.path == "/graphql"
query: Final = _Query.model_validate_json(request.content).variables
bounds: Final = re.search(r"merged:([^ ]+)\.\.([^ ]+)", query.q)
assert bounds is not None
lower, upper = (datetime.fromisoformat(value.replace("Z", "+00:00")) for value in bounds.groups())
assert query.q.count("merged:") == 1
matching: Final = tuple(
_node(index, timestamp) for index, timestamp in enumerate(timestamps) if lower <= timestamp <= upper
)
offset: Final = int(query.after or 0)
next_cursor: Final = str(offset + 100) if offset + 100 < len(matching) else None
return _page(matching[offset : offset + 100], len(matching), next_cursor)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
source: Final = GitHubObserved(ROISettings(github_token=SecretStr("test-only-token")), client)
pulls: Final = await source.pulls("org/repo", start.date(), start.date())
assert tuple(pull.number for pull in pulls) == tuple(range(1001))
assert all(
pull.created_at and (datetime.fromisoformat(pull.merged_at or "") - pull.created_at).total_seconds() == 16
for pull in pulls
)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ("count", "duplicate", "cursor", "partial"))
async def test_incomplete_source_results_fail_instead_of_publishing_understated_counts(failure: str) -> None:
node: Final = _node(1, datetime(2026, 9, 1, tzinfo=timezone.utc))
def respond(request: httpx.Request) -> httpx.Response:
if failure == "partial":
return httpx.Response(200, json={"data": None, "errors": [{"message": "permission denied"}]})
if failure == "duplicate":
return _page((node, node), 2)
if failure == "cursor":
return _page((node,), 2, "repeated")
return _page((node,), 2)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
source: Final = GitHubObserved(ROISettings(), client)
with pytest.raises(SourceError):
await source.pulls("org/repo", date(2026, 9, 1), date(2026, 9, 1))
@pytest.mark.asyncio
async def test_disabled_issue_tracking_is_unknown_instead_of_zero_bugs() -> None:
def respond(request: httpx.Request) -> httpx.Response:
assert request.method == "GET"
return httpx.Response(200, json={"has_issues": False})
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
source: Final = GitHubObserved(ROISettings(), client)
assert await source.issues("org/repo", date(2026, 9, 1), date(2026, 9, 1)) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ("timeout", "unavailable", "rate_limit"))
async def test_read_queries_recover_from_temporary_provider_failures(
failure: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
responses: Final = iter((False, False, True))
node: Final = _node(1, datetime(2026, 9, 1, tzinfo=timezone.utc))
def respond(request: httpx.Request) -> httpx.Response:
if next(responses):
return _page((node,), 1)
if failure == "timeout":
raise httpx.ReadTimeout("scripted timeout", request=request)
return httpx.Response(429 if failure == "rate_limit" else 502)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
source: Final = GitHubObserved(ROISettings(), client)
pulls: Final = await source.pulls("org/repo", date(2026, 9, 1), date(2026, 9, 1))
assert tuple(pull.number for pull in pulls) == (1,)
@pytest.mark.asyncio
async def test_read_retries_stop_after_three_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue()
def respond(request: httpx.Request) -> httpx.Response:
requests.put_nowait(request)
return httpx.Response(503)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
source: Final = GitHubObserved(ROISettings(), client)
with pytest.raises(SourceError, match="HTTP 503"):
await source.pulls("org/repo", date(2026, 9, 1), date(2026, 9, 1))
assert requests.qsize() == 3

25
tests/unit/proxy/roi_calculator/test_gitlab.py
View file

@ -279,3 +279,28 @@ async def test_gitlab_retries_transient_errors_and_checks_merge_request_access()
assert next(statuses, None) is None
finally:
await source.close()
@pytest.mark.asyncio
async def test_observed_issues_preserve_last_second_boundaries_and_disabled_tracking() -> None:
def respond(request: httpx.Request) -> httpx.Response:
if request.url.path.endswith("/projects/org/disabled"):
return httpx.Response(200, json={"id": 2, "path_with_namespace": "org/disabled", "issues_enabled": False})
if request.url.path.endswith("/projects/org/repo"):
return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/repo"})
assert request.url.params["created_before"] == "2026-10-01T00:00:00Z"
return httpx.Response(
200,
json=[
{"iid": 1, "created_at": "2026-09-30T23:59:59.999Z", "labels": ["type::bug"]},
{"iid": 2, "created_at": "2026-10-01T00:00:00Z", "labels": ["bug"]},
],
)
source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond))
try:
issues: Final = await source.issues("org/repo", date(2026, 9, 1), date(2026, 9, 30))
assert issues is not None and tuple(issue.number for issue in issues) == (1,)
assert await source.issues("org/disabled", date(2026, 9, 1), date(2026, 9, 30)) is None
finally:
await source.close()

25
tests/unit/proxy/roi_calculator/test_oauth.py Normal file
View file

@ -0,0 +1,25 @@
from dataclasses import replace
from typing import Final
from pydantic import SecretStr
from litellm.proxy.roi_calculator.oauth import OAuthConfig
def test_app_urls_support_enterprise_and_a_gateway_path_prefix() -> None:
cloud: Final = OAuthConfig(
"github",
"https://api.github.com",
"https://github.com",
"test-client",
SecretStr("test-secret"),
"https://gateway.example.test/proxy",
"test-app",
)
enterprise: Final = replace(cloud, api_url="https://git.example.test/api/v3", base_url="https://git.example.test")
assert cloud.installation_url == cloud.base_url + "/apps/test-app/installations/new"
assert enterprise.installation_url == enterprise.base_url + "/github-apps/test-app/installations/new"
assert enterprise.cookie_path == "/proxy/roi-calculator/observed/oauth"
assert enterprise.redirect_uri.startswith(enterprise.proxy_url + "/roi-calculator/")
assert replace(cloud, provider="gitlab").installation_url is None
assert replace(cloud, app_slug="").installation_url is None

176
tests/unit/proxy/roi_calculator/test_observed_analytics.py Normal file
View file

@ -0,0 +1,176 @@
from datetime import date, datetime, timedelta, timezone
from typing import Final
import pytest
from litellm.proxy.roi_calculator.observed_analytics import (
declared_requester,
merge_hours,
reporting_windows,
summarize_observed,
)
from litellm.types.roi_observed import ObservedData, ObservedIssue, ObservedPeriodData, ObservedPull, ObservedWindow
_NOW: Final = datetime(2026, 10, 3, tzinfo=timezone.utc)
_WINDOW: Final = ObservedWindow(start=date(2026, 9, 5), end=date(2026, 10, 2))
def _pull(login: str, number: int = 1, repo: str = "org/service", **fields: object) -> ObservedPull:
return ObservedPull.model_validate(
{
"repo": repo,
"number": number,
"title": "Ship change",
"url": f"https://github.com/{repo}/pull/{number}",
"author": login,
"created_at": "2026-09-10T00:00:00Z",
"merged_at": "2026-09-10T00:01:19Z",
**fields,
}
)
def _data(current: ObservedPeriodData, previous: ObservedPeriodData | None = None) -> ObservedData:
empty: Final = ObservedPeriodData(window=_WINDOW, pulls=(), issues=(), spend=())
return ObservedData(
source_provider="github",
source_api_url="https://api.github.com",
repos=("org/service",),
captured_at=_NOW,
gateway_emails=("ari@example.test", "bea@example.test"),
current=current,
previous=previous or empty,
last_year=empty,
)
def test_multiple_accounts_share_one_cost_denominator_and_pr_numbers_are_scoped_to_repositories() -> None:
direct: Final = _pull("ari", profile_email="ari@example.test")
alternate: Final = _pull("old-ari", repo="org/other")
agent: Final = _pull("devin-ai[bot]", 3, agent=True, requester="old-ari")
unowned: Final = _pull("devin-ai[bot]", 4, agent=True)
period: Final = ObservedPeriodData(
window=_WINDOW,
pulls=(direct, alternate, agent, unowned),
issues=(),
spend=({"email": "ari@example.test", "spend": 90.0, "date": "2026-09-10", "user_id": "ari", "requests": 1},),
)
report: Final = summarize_observed(_data(period), {"old-ari": "ari@example.test"})
assert len(report.people) == 1
person: Final = report.people[0]
assert person.logins == ("ari", "old-ari")
assert person.periods.current.pr_urls == (direct.url, alternate.url, agent.url)
assert (person.periods.current.direct_authored, person.periods.current.declared_agent_owned) == (2, 1)
assert person.periods.current.recorded_spend_per_attributed_pr == 30.0
assert person.periods.current.prs_per_week == 0.75
assert (report.periods.current.merged_prs, report.periods.current.matched_internal_prs) == (4, 3)
assert report.periods.current.agents_without_requester == 1
def test_missing_spend_stays_unknown_and_a_recorded_zero_stays_zero() -> None:
period: Final = ObservedPeriodData(
window=_WINDOW,
pulls=(_pull("ari"), _pull("bea", 2)),
issues=None,
spend=({"email": "bea@example.test", "spend": 0.0, "date": "2026-09-10", "user_id": "bea", "requests": 1},),
)
report: Final = summarize_observed(_data(period), {"ari": "ari@example.test", "bea": "bea@example.test"})
assert tuple(person.periods.current.recorded_spend_per_attributed_pr for person in report.people) == (None, 0.0)
assert tuple(person.periods.current.spend_observation for person in report.people) == (
"no_records",
"records_present",
)
assert report.periods.current.new_bug_labeled_issues is None
assert report.periods.previous.new_bug_labeled_issues == 0
assert report.people[0].periods.previous.recorded_spend_per_attributed_pr is None
def test_manual_links_override_automatic_matches_and_removal_suppresses_rematching() -> None:
period: Final = ObservedPeriodData(
window=_WINDOW, pulls=(_pull("ari", profile_email="ari@example.test"),), issues=(), spend=()
)
data: Final = _data(period)
assert summarize_observed(data, {}).people[0].email == "ari@example.test"
assert summarize_observed(data, {"ari": "bea@example.test"}).people[0].email == "bea@example.test"
removed: Final = summarize_observed(data, {}, ("ari",))
assert removed.people == ()
assert removed.unmatched_logins == ("ari",)
assert summarize_observed(data, {"ari": "bea@example.test"}, ("ari",)).people[0].email == "bea@example.test"
def test_conflicting_public_emails_do_not_silently_choose_an_owner() -> None:
current: Final = ObservedPeriodData(
window=_WINDOW, pulls=(_pull("ari", profile_email="ari@example.test"),), issues=(), spend=()
)
previous: Final = current.model_copy(update={"pulls": (_pull("ari", profile_email="bea@example.test"),)})
report: Final = summarize_observed(_data(current, previous), {})
assert report.people == ()
assert report.unmatched_logins == ("ari",)
def test_quality_counts_labelled_issues_once_and_does_not_infer_bugs_from_pr_titles() -> None:
period: Final = ObservedPeriodData(
window=_WINDOW,
pulls=(_pull("ari", title="fix: critical bug"), _pull("ari", 2, title='Revert "change"')),
issues=tuple(
ObservedIssue(repo="org/service", number=index, created_at=_NOW, labels=labels)
for index, labels in enumerate(
(
("BUG", "kind:bug"),
("type::bug", "type::regression"),
("debug",),
)
)
),
spend=(),
)
report: Final = summarize_observed(_data(period), {})
assert (
report.periods.current.new_bug_labeled_issues,
report.periods.current.new_regression_labeled_issues,
report.periods.current.explicitly_titled_revert_prs,
) == (2, 1, 1)
@pytest.mark.parametrize(
"created,merged,expected",
(
(None, "2026-09-10T00:00:16Z", None),
("2026-09-10T00:00:00Z", "2026-09-10T00:00:16Z", 16 / 3600),
("2026-09-10T00:00:00Z", "2026-09-10T00:00:00Z", 0),
("2026-09-10T00:00:01Z", "2026-09-10T00:00:00Z", None),
("2026-09-10T00:00:00", "2026-09-10T00:00:16Z", None),
),
)
def test_merge_duration_preserves_seconds_and_rejects_invalid_intervals(
created: str | None, merged: str, expected: float | None
) -> None:
assert merge_hours(_pull("ari", created_at=created, merged_at=merged)) == expected
@pytest.mark.parametrize(
"now", (datetime(2024, 3, 1, tzinfo=timezone.utc), _NOW, _NOW.replace(tzinfo=timezone(timedelta(hours=14))))
)
@pytest.mark.parametrize("days", (1, 7, 28, 90, 366))
def test_reporting_windows_have_equal_lengths_and_exclude_today(now: datetime, days: int) -> None:
current, previous, yearly = reporting_windows(now, days)
assert all((window.end - window.start).days + 1 == days for window in (current, previous, yearly))
assert current.end == now.astimezone(timezone.utc).date() - timedelta(days=1)
assert previous.end == current.start - timedelta(days=1)
assert yearly.end.year == current.end.year - 1
assert yearly.end.month == current.end.month
@pytest.mark.parametrize(
"author,body,expected",
(
("devin-ai[bot]", "Requested by: @Ari", "ari"),
("devin-ai-integration", "Requested by: @Ari", "ari"),
("devin-ai", "Requested by: @Ari", "ari"),
("human", "Requested by: @ari", ""),
("devin-ai[bot]", "Requested by: @ari\nRequested by: @bea", ""),
("devin-ai[bot]", "Mentions @ari", ""),
),
)
def test_agent_ownership_requires_one_explicit_requester(author: str, body: str, expected: str) -> None:
assert declared_requester(author, body) == expected

140
tests/unit/proxy/roi_calculator/test_observed_sync.py Normal file
View file

@ -0,0 +1,140 @@
from datetime import date, datetime, timezone
from typing import Final, Literal
import httpx
import pytest
from pydantic import BaseModel, SecretStr
from litellm.proxy.roi_calculator.observed_analytics import summarize_observed
from litellm.proxy.roi_calculator.observed_sync import collect_observed
from litellm.proxy.roi_calculator.source import repository_tag
from litellm.types.roi_calculator import ROIBranchSpend, ROISettings, ROISpendRecord
class _Variables(BaseModel):
q: str
class _Query(BaseModel):
variables: _Variables
@pytest.mark.asyncio
@pytest.mark.parametrize("provider", ("github", "gitlab"))
@pytest.mark.parametrize("days", (7, 28, 90))
@pytest.mark.parametrize("include_disabled_repo", (False, True))
async def test_live_provider_metadata_reaches_people_quality_durations_and_branch_spend_without_an_estimator(
provider: Literal["github", "gitlab"],
days: int,
include_disabled_repo: bool,
) -> None:
settings: Final = ROISettings(
source_provider=provider,
repos=("org/repo", "org/disabled") if include_disabled_repo else ("org/repo",),
github_token=SecretStr("source-test-token"),
gitlab_token=SecretStr("source-test-token"),
identity_map={"old-ari": "ari@example.test"},
)
tag: Final = repository_tag(settings, "org/repo")
async def spend(start: date, end: date) -> tuple[ROISpendRecord, ...]:
return ({"date": str(start), "user_id": "ari", "email": "ari@example.test", "spend": 30.0, "requests": 10},)
async def users() -> frozenset[str]:
return frozenset(("ari@example.test",))
async def branches(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]:
assert repos == tuple(sorted(repository_tag(settings, repo) for repo in settings.repos))
return (ROIBranchSpend(repo=tag, branch="fix/parser", spend=5.0, requests=2),)
def respond(request: httpx.Request) -> httpx.Response:
path: Final = request.url.path
if path == "/graphql":
query: Final = _Query.model_validate_json(request.content).variables.q
if "repo:org/disabled " in query:
return httpx.Response(
200, json={"data": {"search": {"issueCount": 0, "nodes": [], "pageInfo": {"hasNextPage": False}}}}
)
kind: Final = "pull" if "is:pr" in query else "issue"
start: Final = query.split("merged:" if kind == "pull" else "created:")[1][:10]
node: Final = {
"number": 1,
"url": "https://github.com/org/repo/pull/1",
"title": "Change",
"createdAt": f"{start}T12:00:00Z",
"updatedAt": f"{start}T12:00:16Z",
"mergedAt": f"{start}T12:00:16Z",
"author": {"login": "devin-ai-integration", "__typename": "Bot"},
"body": "Requested by: @old-ari",
"headRefName": "fix/parser",
"headRepository": {"nameWithOwner": "org/repo"},
"labels": {"nodes": [{"name": "bug"}], "pageInfo": {"hasNextPage": False}},
}
return httpx.Response(
200, json={"data": {"search": {"issueCount": 1, "nodes": [node], "pageInfo": {"hasNextPage": False}}}}
)
if path == "/repos/org/repo":
return httpx.Response(200, json={"has_issues": True})
if path == "/repos/org/disabled":
return httpx.Response(200, json={"has_issues": False})
if path.endswith("/projects/org/disabled"):
return httpx.Response(200, json={"id": 2, "path_with_namespace": "org/disabled", "issues_enabled": False})
if path.endswith("/projects/2/merge_requests"):
return httpx.Response(200, json=[])
if path.endswith("/projects/org/repo"):
return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/repo"})
if path.endswith("/merge_requests"):
start: Final = request.url.params["merged_after"][:10]
return httpx.Response(
200,
json=[
{
"iid": 1,
"web_url": "https://gitlab.com/org/repo/-/merge_requests/1",
"title": "Change",
"author": {"username": "old-ari"},
"created_at": f"{start}T12:00:00Z",
"updated_at": f"{start}T12:00:16Z",
"merged_at": f"{start}T12:00:16Z",
"source_branch": "fix/parser",
"source_project_id": 1,
}
],
)
if path.endswith("/issues"):
start: Final = request.url.params["created_after"][:10]
return httpx.Response(200, json=[{"iid": 1, "created_at": f"{start}T12:00:00Z", "labels": ["bug"]}])
raise AssertionError(f"Unexpected API request: {request.method} {path}")
data: Final = await collect_observed(
settings,
spend,
users,
branches,
datetime(2026, 10, 3, tzinfo=timezone.utc),
lambda stage, done, total: None,
httpx.MockTransport(respond),
days=days,
)
assert all(
(period.window.end - period.window.start).days + 1 == days
for period in (data.current, data.previous, data.last_year)
)
report: Final = summarize_observed(data, settings.identity_map)
person: Final = report.people[0].periods.current
assert (person.merged_prs, person.gateway_recorded_spend, person.recorded_spend_per_attributed_pr) == (
1,
30.0,
30.0,
)
assert person.median_merge_hours == 16 / 3600
assert person.declared_agent_owned == (1 if provider == "github" else 0)
assert tuple(
window.merged_prs for window in (report.periods.current, report.periods.previous, report.periods.last_year)
) == (1, 1, 1)
assert tuple(
period.new_bug_labeled_issues
for period in (report.periods.current, report.periods.previous, report.periods.last_year)
) == (1, 1, 1)
assert report.pulls.current[0].branch_cost.spend == 5.0
assert report.unlinked_branches == ()

136
tests/unit/proxy/roi_calculator/test_observed_workspace.py Normal file
View file

@ -0,0 +1,136 @@
from datetime import date, datetime, timezone
from typing import Final
import pytest
from litellm.proxy.roi_calculator.github import SourceError
from litellm.proxy.roi_calculator.observed_workspace import (
combine_observed,
scoped_data,
source_details,
summarize_workspace,
)
from litellm.proxy.roi_calculator.settings import StoredConnection
from litellm.types.roi_calculator import ROIBranchSpend, ROISettings
from litellm.types.roi_observed import ObservedData, ObservedIssue, ObservedPeriodData, ObservedPull, ObservedWindow
def _source(settings: ROISettings, issues: tuple[ObservedIssue, ...] | None = ()) -> ObservedData:
host: Final = "github.com" if settings.source_provider == "github" else "gitlab.com"
period: Final = ObservedPeriodData(
window=ObservedWindow(start=date(2026, 9, 1), end=date(2026, 9, 28)),
pulls=tuple(
ObservedPull(
repo=repo,
number=1,
title="Change",
url=f"https://{host}/{repo}/pull/1",
author="ari",
created_at=datetime(2026, 9, 10, 0, 0, 0, tzinfo=timezone.utc),
merged_at=datetime(2026, 9, 10, 0, 0, 30, tzinfo=timezone.utc),
source_repo=f"{host}/{repo}",
source_branch="feature/one",
)
for repo in settings.repos
),
issues=issues,
spend=({"date": "2026-09-10", "user_id": "ari", "email": "ari@example.test", "spend": 60.0, "requests": 3},),
branch_spend=tuple(
ROIBranchSpend(repo=f"{host}/{repo}", branch="feature/one", spend=2, requests=1) for repo in settings.repos
),
)
data: Final = ObservedData(
source_provider=settings.source_provider,
source_api_url=settings.source_api_url,
repos=settings.repos,
captured_at=datetime(2026, 9, 29, tzinfo=timezone.utc),
gateway_emails=("ari@example.test", "bea@example.test"),
current=period,
previous=period,
last_year=period,
)
return scoped_data(data, source_details(settings))
@pytest.mark.parametrize("same_person", (True, False))
def test_multiple_repos_and_providers_scope_usernames_and_count_spend_once(same_person: bool) -> None:
github: Final = ROISettings(repos=("org/service", "org/docs"))
gitlab: Final = ROISettings(source_provider="gitlab", repos=("org/service",))
combined: Final = combine_observed(
(_source(github), _source(gitlab)), ("github.com/org/service", "github.com/org/docs", "gitlab.com/org/service")
)
report: Final = summarize_workspace(
combined,
(
StoredConnection(
source_provider="github", api_url=github.source_api_url, identity_map={"ari": "ari@example.test"}
),
StoredConnection(
source_provider="gitlab",
api_url=gitlab.source_api_url,
identity_map={"ari": "ari@example.test" if same_person else "bea@example.test"},
),
),
)
person: Final = next(person for person in report.people if person.email == "ari@example.test")
assert report.source_provider == "mixed"
assert report.periods.current.merged_prs == 3
assert report.periods.current.matched_users_recorded_spend == 60
assert person.periods.current.merged_prs == (3 if same_person else 2)
assert person.periods.current.gateway_recorded_spend == 60
assert person.periods.current.recorded_spend_per_attributed_pr == (20 if same_person else 30)
assert len(person.accounts) == (2 if same_person else 1)
assert all(pull.branch_cost.spend == 2 for pull in report.pulls.current)
assert report.unlinked_branches == ()
def test_empty_repository_is_a_successful_zero_activity_report() -> None:
settings: Final = ROISettings(repos=())
data: Final = combine_observed((_source(settings),), ("org/empty",))
report: Final = summarize_workspace(data, ())
assert report.periods.current.merged_prs == 0
assert report.periods.current.new_bug_labeled_issues == 0
assert report.periods.current.median_merge_hours is None
assert report.people == () and report.pulls.current == ()
@pytest.mark.parametrize(
("issues", "expected"),
(
(None, None),
((), 0),
(
(
ObservedIssue(
repo="org/service",
number=1,
created_at=datetime(2026, 9, 10, tzinfo=timezone.utc),
labels=("bug", "regression"),
),
),
1,
),
),
)
def test_disabled_tracking_does_not_hide_other_connections_quality_counts(
issues: tuple[ObservedIssue, ...] | None, expected: int | None
) -> None:
github: Final = _source(ROISettings(repos=("org/docs",)), issues=None)
gitlab: Final = _source(ROISettings(source_provider="gitlab", repos=("org/service",)), issues=issues)
report: Final = summarize_workspace(combine_observed((github, gitlab), ("org/docs", "org/service")), ())
assert tuple(
(period.new_bug_labeled_issues, period.new_regression_labeled_issues)
for period in (report.periods.current, report.periods.previous, report.periods.last_year)
) == ((expected, expected),) * 3
assert report.periods.current.merged_prs == 2
def test_duplicate_connection_cannot_double_count_a_merged_change() -> None:
data: Final = _source(ROISettings(repos=("org/service",)))
with pytest.raises(SourceError, match="more than one connection"):
combine_observed((data, data), data.repos)
def test_github_repository_selection_deduplicates_case_variants() -> None:
settings: Final = ROISettings(repos=("Org/Service", "org/service", "org/docs", "org/docs.git"))
assert settings.repos == ("Org/Service", "org/docs")

13
tests/unit/test_assert_ci_coverage.py
View file

@ -73,6 +73,19 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path:
assert [(finding.subject, finding.detail) for finding in findings] == [
(test_path, "integration contract is also selected by GitHub Actions")
]
github_path: Final = "tests/integration/management/test_github_contract.py"
(tmp_path / github_path).write_text("def test_contract(): pass\n")
runner: Final = tmp_path / "tests/integration/run.py"
runner.write_text(runner.read_text() + f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n")
workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}}))
github_owned, github_findings = coverage._integration_ownership(tmp_path)
assert github_owned == frozenset({test_path, github_path})
assert github_findings == ()
workflow.write_text(yaml.safe_dump({"jobs": {}}))
_, missing_invocation = coverage._integration_ownership(tmp_path)
assert [(finding.subject, finding.detail) for finding in missing_invocation] == [
(github_path, "GitHub-owned integration contract has no invoking workflow")
]
def test_an_ancestor_directory_covers_a_file_but_does_not_name_it():

88
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.integration.test.tsx Normal file
View file

@ -0,0 +1,88 @@
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, expect, it, vi } from "vitest";
import ObservedAccounts from "./ObservedAccounts";
afterEach(() => vi.unstubAllGlobals());
it("links several usernames on both providers to one email in a single save", async () => {
const writes: unknown[] = [];
const saved = vi.fn();
vi.stubGlobal(
"fetch",
vi.fn(async (_input: string, init: RequestInit) => {
if (init.method === "PUT") {
writes.push(JSON.parse(String(init.body)));
return Response.json({ report: null });
}
const identities = {
gateway_emails: ["ari@example.test"],
identity_map: { old: "ari@example.test" },
unmatched_logins: ["new"],
connections: [
{
id: "github-id",
source_provider: "github",
api_url: "https://api.github.com",
identity_map: { old: "ari@example.test" },
unmatched_logins: ["new"],
},
{
id: "gitlab-id",
source_provider: "gitlab",
api_url: "https://gitlab.com/api/v4",
identity_map: {},
unmatched_logins: ["new"],
},
],
};
return Response.json(identities);
}),
);
const user = userEvent.setup();
render(<ObservedAccounts accessToken="gateway-test-token" people={[]} onClose={vi.fn()} onSaved={saved} />);
await screen.findByLabelText(/GitHub usernames/);
fireEvent.change(screen.getByLabelText("Internal email"), { target: { value: "ari@example.test" } });
expect(screen.getByLabelText(/GitHub usernames/)).toHaveValue("old");
fireEvent.change(screen.getByLabelText(/GitHub usernames/), { target: { value: "@Old, new, NEW" } });
fireEvent.change(screen.getByLabelText(/GitLab usernames/), { target: { value: "new" } });
await user.click(screen.getByRole("button", { name: "Save accounts" }));
await waitFor(() => expect(saved).toHaveBeenCalledOnce());
expect(writes).toEqual([
{
email: "ari@example.test",
accounts: [
{ connection_id: "github-id", login: "old" },
{ connection_id: "github-id", login: "new" },
{ connection_id: "gitlab-id", login: "new" },
],
},
]);
});
it("keeps a conflicting link editable", async () => {
const saved = vi.fn();
vi.stubGlobal(
"fetch",
vi.fn(async (_input: string, init: RequestInit) =>
init.method === "PUT"
? Response.json({ detail: "An account is already linked to another email. Unlink it first" }, { status: 409 })
: Response.json({ gateway_emails: ["ari@example.test"], identity_map: {}, unmatched_logins: [] }),
),
);
const user = userEvent.setup();
render(
<ObservedAccounts
accessToken="gateway-test-token"
people={[]}
initialEmail="ari@example.test"
onClose={vi.fn()}
onSaved={saved}
/>,
);
fireEvent.change(screen.getByLabelText("Source usernames"), { target: { value: "old, new" } });
const button = screen.getByRole("button", { name: "Save accounts" });
await waitFor(() => expect(button).toBeEnabled());
await user.click(button);
expect(await screen.findByRole("alert")).toHaveTextContent("An account is already linked to another email");
expect(screen.getByLabelText("Source usernames")).toHaveValue("old, new");
expect(saved).not.toHaveBeenCalled();
});

209
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.tsx Normal file
View file

@ -0,0 +1,209 @@
"use client";
import { useEffect, useState } from "react";
import { z } from "zod";
import { apiClient } from "@/components/networking";
import { extractProxyErrorMessage } from "@/lib/http/client";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog";
import { accountLogins, type ObservedPerson } from "./observedData";
const connectionIdentityFields = {
id: z.string(),
source_provider: z.enum(["github", "gitlab"]),
api_url: z.string(),
identity_map: z.record(z.string(), z.string()),
unmatched_logins: z.array(z.string()),
};
const identitiesFields = {
gateway_emails: z.array(z.string()),
identity_map: z.record(z.string(), z.string()),
unmatched_logins: z.array(z.string()),
connections: z.array(z.object(connectionIdentityFields)).optional(),
};
const identitiesSchema = z.object(identitiesFields);
function matches(identities: z.infer<typeof identitiesSchema>, email: string, people: ObservedPerson[]) {
const person = people.find((entry) => entry.email === email);
return Object.fromEntries(
(identities.connections ?? []).map((entry) => [
entry.id,
[
...new Set([
...Object.entries(entry.identity_map)
.filter(([, address]) => address === email)
.map(([login]) => login),
...(person?.accounts ?? [])
.filter((account) => account.connection_id === entry.id)
.map((account) => account.login),
]),
].join(", "),
]),
);
}
export default function ObservedAccounts({
accessToken,
people,
initialEmail = "",
onClose,
onSaved,
}: {
accessToken: string;
people: ObservedPerson[];
initialEmail?: string;
onClose: () => void;
onSaved: () => void;
}) {
const [identities, setIdentities] = useState<z.infer<typeof identitiesSchema> | null>(null);
const [email, setEmail] = useState(initialEmail);
const [logins, setLogins] = useState(people.find((person) => person.email === initialEmail)?.logins.join(", ") ?? "");
const [linked, setLinked] = useState<Record<string, string>>({});
const [saving, setSaving] = useState(false);
const [error, setError] = useState("");
useEffect(() => {
const controller = new AbortController();
apiClient
.get<unknown>("/roi-calculator/observed/identities", { accessToken, signal: controller.signal })
.then((data) => {
if (!controller.signal.aborted) {
const parsed = identitiesSchema.parse(data);
setIdentities(parsed);
setLinked(matches(parsed, initialEmail, people));
}
})
.catch((reason: unknown) => {
if (!controller.signal.aborted) setError(extractProxyErrorMessage(reason));
});
return () => controller.abort();
}, [accessToken, initialEmail, people]);
function selectEmail(value: string) {
setEmail(value);
if (identities) setLinked(matches(identities, value, people));
const automatic = people.find((person) => person.email === value)?.logins ?? [];
const manual = Object.entries(identities?.identity_map ?? {})
.filter(([, address]) => address === value)
.map(([login]) => login);
setLogins([...new Set([...automatic, ...manual])].join(", "));
}
async function save() {
setSaving(true);
setError("");
try {
await apiClient.put<unknown>("/roi-calculator/observed/identities", {
accessToken,
body: {
email: email.trim().toLowerCase(),
...(identities?.connections?.length
? {
accounts: identities.connections.flatMap((entry) =>
accountLogins(linked[entry.id] ?? "").map((login) => ({ connection_id: entry.id, login })),
),
}
: { logins: accountLogins(logins) }),
},
});
onSaved();
onClose();
} catch (reason) {
setError(extractProxyErrorMessage(reason));
} finally {
setSaving(false);
}
}
return (
<Dialog
open
onOpenChange={(open) => {
if (!open) onClose();
}}
>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>Link accounts</DialogTitle>
<DialogDescription>Match one internal user to all their source accounts</DialogDescription>
</DialogHeader>
<div className="space-y-4 py-2">
<div className="space-y-2">
<label htmlFor="identity-email" className="text-sm font-medium">
Internal email
</label>
<Input
id="identity-email"
disabled={!identities}
list="roi-internal-emails"
value={email}
placeholder="Choose an internal user"
onChange={(event) => selectEmail(event.target.value)}
/>
<datalist id="roi-internal-emails">
{identities?.gateway_emails.map((address) => <option key={address} value={address} />)}
</datalist>
</div>
{identities?.connections?.length ? (
identities.connections.map((entry) => (
<div key={entry.id} className="space-y-2">
<label htmlFor={`identity-${entry.id}`} className="text-sm font-medium">
{entry.source_provider === "github" ? "GitHub" : "GitLab"} usernames
<span className="ml-2 text-xs font-normal text-muted-foreground">{new URL(entry.api_url).host}</span>
</label>
<Input
id={`identity-${entry.id}`}
value={linked[entry.id] ?? ""}
onChange={(event) => setLinked({ ...linked, [entry.id]: event.target.value })}
placeholder="current-account, old-account"
/>
{entry.unmatched_logins.length > 0 && (
<details className="text-xs text-muted-foreground">
<summary className="cursor-pointer">{entry.unmatched_logins.length} unmatched accounts</summary>
<div className="mt-2 max-h-32 overflow-auto flex flex-wrap gap-1">
{entry.unmatched_logins.map((login) => (
<Button
key={login}
size="sm"
variant="outline"
onClick={() =>
setLinked({
...linked,
[entry.id]: [...new Set([...accountLogins(linked[entry.id] ?? ""), login])].join(", "),
})
}
>
{login}
</Button>
))}
</div>
</details>
)}
</div>
))
) : (
<div className="space-y-2">
<label htmlFor="identity-logins" className="text-sm font-medium">
Source usernames
</label>
<Input
id="identity-logins"
value={logins}
onChange={(event) => setLogins(event.target.value)}
placeholder="current-account, old-account"
/>
</div>
)}
<p className="text-xs text-muted-foreground">
Separate accounts with commas. Their merged changes are combined, and gateway spend is counted once
</p>
{error && (
<p role="alert" className="text-sm text-destructive">
{error}
</p>
)}
<Button className="w-full" disabled={!identities || !email.trim() || saving} onClick={save}>
{saving ? "Saving…" : "Save accounts"}
</Button>
</div>
</DialogContent>
</Dialog>
);
}

198
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.integration.test.tsx Normal file
View file

@ -0,0 +1,198 @@
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, describe, expect, it, vi } from "vitest";
import ObservedConnections from "./ObservedConnections";
import type { ObservedSettings } from "./observedData";
const settings: ObservedSettings = {
id: "initial-github",
source_provider: "github",
api_url: "https://api.github.com",
repos: [],
has_token: false,
connection_type: "token",
update_interval_minutes: 1440,
ready: false,
};
const app = { configured: true, api_url: null, callback_url: null };
afterEach(() => vi.unstubAllGlobals());
describe("observed ROI connections", () => {
it("identifies the saved connection when editing its host", async () => {
const writes: unknown[] = [];
const existing = { ...settings, id: "saved-github", has_token: true, repos: ["org/service"], ready: true };
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
const path = new URL(input, "http://localhost").pathname;
if (path.endsWith("/apps")) return Response.json({ github: app, gitlab: app });
if (path.endsWith("/repositories")) return Response.json({ repositories: [], has_more: false });
if (path.endsWith("/settings")) {
writes.push(JSON.parse(String(init.body)));
return Response.json({ ...existing, id: "enterprise-github", api_url: "https://git.example.test/api/v3" });
}
throw new Error(path);
}),
);
const user = userEvent.setup();
render(
<ObservedConnections
accessToken="gateway-test-token"
settings={{ ...existing, connections: [existing] }}
onClose={vi.fn()}
onSaved={vi.fn()}
/>,
);
await user.click(screen.getByRole("button", { name: "Edit GitHub api.github.com" }));
await user.click(screen.getByRole("button", { name: "Change connection" }));
await user.click(screen.getByText("Self-hosted instance"));
fireEvent.change(screen.getByLabelText("API URL"), { target: { value: "https://git.example.test/api/v3" } });
fireEvent.change(screen.getByLabelText("GitHub access token"), { target: { value: "enterprise-test-token" } });
await user.click(screen.getByRole("button", { name: "Continue" }));
expect(await screen.findByRole("heading", { name: "Choose repositories" })).toBeInTheDocument();
expect(writes).toEqual([
{
connection_id: "saved-github",
source_provider: "github",
api_url: "https://git.example.test/api/v3",
token: "enterprise-test-token",
repos: [],
update_interval_minutes: 1440,
},
]);
});
it("starts a GitHub installation when switching from a GitLab app connection", async () => {
const starts = vi.fn();
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
const url = new URL(input, "http://localhost");
if (url.pathname.endsWith("/apps"))
return Response.json({ github: { ...app, can_install: true }, gitlab: app });
if (url.pathname.endsWith("/repositories")) return Response.json({ repositories: [], has_more: false });
starts(url.searchParams.get("install"), init.credentials);
return Response.json({ detail: "Authorization test stopped before redirect" }, { status: 502 });
}),
);
const user = userEvent.setup();
const connected: ObservedSettings = {
...settings,
source_provider: "gitlab",
connection_type: "app",
has_token: true,
};
render(<ObservedConnections accessToken="test-token" settings={connected} onClose={vi.fn()} onSaved={vi.fn()} />);
await user.click(screen.getByRole("button", { name: "Change connection" }));
await user.click(screen.getByRole("button", { name: "GitHub", exact: true }));
const connect = screen.getByRole("button", { name: "Connect GitHub", exact: true });
await waitFor(() => expect(connect).toBeEnabled());
await user.click(connect);
expect(await screen.findByRole("alert")).toHaveTextContent("Authorization test stopped before redirect");
expect(starts).toHaveBeenCalledWith("true", "include");
});
it.each(["GitHub", "GitLab"] as const)(
"connects %s with a token, saves repositories, and starts a sync",
async (label) => {
const provider = label === "GitHub" ? "github" : "gitlab";
const apiUrl = provider === "github" ? "https://api.github.com" : "https://gitlab.com/api/v4";
const saved = vi.fn();
const closed = vi.fn();
const writes: { path: string; body: unknown }[] = [];
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
const path = new URL(input, "http://localhost").pathname;
if (init.method === "PUT" || init.method === "POST")
writes.push({ path, body: init.body ? JSON.parse(String(init.body)) : undefined });
if (path.endsWith("/apps")) return Response.json({ github: app, gitlab: app });
if (path.endsWith("/repositories"))
return Response.json({
repositories: [{ name: "org/service", visibility: "private", archived: false }],
has_more: false,
});
if (path.endsWith("/settings")) {
const request = JSON.parse(String(init.body)) as { repos: string[] };
const connected = {
...settings,
id: `saved-${provider}`,
source_provider: provider,
api_url: apiUrl,
has_token: true,
repos: request.repos,
ready: request.repos.length > 0,
};
return Response.json(connected);
}
if (path.endsWith("/sync")) return Response.json({ running: true }, { status: 202 });
throw new Error(path);
}),
);
const user = userEvent.setup();
render(
<ObservedConnections accessToken="gateway-test-token" settings={settings} onClose={closed} onSaved={saved} />,
);
await user.click(screen.getByRole("button", { name: label, exact: true }));
await user.click(screen.getByRole("button", { name: "Access token", exact: true }));
fireEvent.change(screen.getByLabelText(`${label} access token`), { target: { value: "source-test-token" } });
await user.click(screen.getByRole("button", { name: "Continue" }));
await user.click(await screen.findByRole("checkbox", { name: /org\/service/ }));
await user.click(screen.getByRole("button", { name: "Save and sync" }));
await waitFor(() => expect(saved).toHaveBeenCalledOnce());
expect(closed).toHaveBeenCalledOnce();
expect(writes).toEqual([
{
path: "/roi-calculator/observed/settings",
body: {
source_provider: provider,
api_url: apiUrl,
token: "source-test-token",
repos: [],
update_interval_minutes: 1440,
},
},
{
path: "/roi-calculator/observed/settings",
body: {
connection_id: `saved-${provider}`,
source_provider: provider,
api_url: apiUrl,
repos: ["org/service"],
update_interval_minutes: 1440,
},
},
{ path: "/roi-calculator/observed/sync", body: undefined },
]);
},
);
it.each(["GitHub", "GitLab"] as const)(
"starts %s app authorization with a browser cookie and displays provider failures",
async (label) => {
const provider = label.toLowerCase();
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
if (String(input).endsWith("/apps")) return Response.json({ github: app, gitlab: app });
expect(String(input)).toContain(`/oauth/${provider}/start`);
expect(init.credentials).toBe("include");
expect(init.method).toBe("POST");
return Response.json({ detail: "Provider unavailable. Try again" }, { status: 502 });
}),
);
const user = userEvent.setup();
render(
<ObservedConnections
accessToken="gateway-test-token"
settings={settings}
onClose={vi.fn()}
onSaved={vi.fn()}
/>,
);
await user.click(screen.getByRole("button", { name: label, exact: true }));
const button = await screen.findByRole("button", { name: `Connect ${label}`, exact: true });
await waitFor(() => expect(button).toBeEnabled());
await user.click(button);
expect(await screen.findByRole("alert")).toHaveTextContent("Provider unavailable. Try again");
expect(button).toBeEnabled();
},
);
});

606
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.tsx Normal file
View file

@ -0,0 +1,606 @@
"use client";
import { useEffect, useState } from "react";
import { Github, Gitlab, ArrowLeft, KeyRound } from "lucide-react";
import { z } from "zod";
import { apiClient } from "@/components/networking";
import { extractProxyErrorMessage } from "@/lib/http/client";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog";
import {
observedSettingsSchema,
repositoryNames,
type ObservedSettings,
type ObservedConnection,
} from "./observedData";
const appFields = {
configured: z.boolean(),
can_install: z.boolean().optional().default(false),
api_url: z.string().nullable(),
callback_url: z.string().nullable(),
};
const appSchema = z.object(appFields);
const appsSchema = z.object({ github: appSchema, gitlab: appSchema });
const repositoriesSchema = z.object({
repositories: z.array(z.object({ name: z.string(), visibility: z.string(), archived: z.boolean() })),
has_more: z.boolean(),
});
const defaultUrl = { github: "https://api.github.com", gitlab: "https://gitlab.com/api/v4" };
function preferredConnectionMethod(selected: "app" | "token" | null, configured: boolean | undefined) {
return selected ?? (configured ? "app" : "token");
}
function TokenFields({
label,
provider,
token,
onToken,
apiUrl,
onUrl,
hasToken,
}: {
label: string;
provider: ObservedSettings["source_provider"];
token: string;
onToken: (value: string) => void;
apiUrl: string;
onUrl: (value: string) => void;
hasToken: boolean;
}) {
return (
<>
<label htmlFor="roi-source-token" className="block text-sm font-medium">
{label} access token
</label>
<Input
id="roi-source-token"
type="password"
autoComplete="off"
value={token}
onChange={(event) => onToken(event.target.value)}
placeholder={hasToken ? "Leave blank to keep the saved token" : "Optional for public repositories"}
/>
<p className="text-xs text-muted-foreground">
{provider === "github"
? "Fine-grained token: read access to pull requests, issues, and metadata"
: "Token with read_api scope"}
</p>
<details className="text-sm">
<summary className="cursor-pointer">Self-hosted instance</summary>
<label htmlFor="roi-source-url" className="mt-3 block text-xs">
API URL
</label>
<Input id="roi-source-url" value={apiUrl} onChange={(event) => onUrl(event.target.value)} />
</details>
</>
);
}
function AppMessage({ configured, label }: { configured: boolean; label: string }) {
return (
<p className="text-sm text-muted-foreground">
{configured
? `You’ll authorize ${label}, then choose repositories`
: `Register the ${label} app in gateway settings, or connect with a token`}
</p>
);
}
function RepositoryChoices({
available,
repos,
setRepos,
query,
setQuery,
page,
setPage,
}: {
available: z.infer<typeof repositoriesSchema> | null;
repos: string;
setRepos: (value: string) => void;
query: string;
setQuery: (value: string) => void;
page: number;
setPage: (value: number) => void;
}) {
return (
<>
<Input
aria-label="Find repositories"
value={query}
onChange={(event) => {
setQuery(event.target.value);
setPage(1);
}}
placeholder="Find repositories…"
/>
<div className="max-h-48 overflow-y-auto rounded-lg border divide-y">
{!available && (
<p role="status" className="p-3 text-sm text-muted-foreground">
Loading repositories…
</p>
)}
{available?.repositories.length === 0 && (
<p className="p-3 text-sm text-muted-foreground">No repositories found</p>
)}
{available?.repositories
.filter((repo) => !repo.archived)
.map((repo) => (
<label key={repo.name} className="flex cursor-pointer items-center gap-3 p-3 text-sm">
<input
type="checkbox"
checked={repositoryNames(repos).includes(repo.name)}
onChange={(event) =>
setRepos(
(event.target.checked
? [...new Set([...repositoryNames(repos), repo.name])]
: repositoryNames(repos).filter((name) => name !== repo.name)
).join(", "),
)
}
/>
<span className="flex-1">{repo.name}</span>
<span className="text-xs text-muted-foreground">{repo.visibility}</span>
</label>
))}
</div>
<div className="flex justify-between">
<Button size="sm" variant="ghost" disabled={page === 1} onClick={() => setPage(page - 1)}>
Previous
</Button>
<Button size="sm" variant="ghost" disabled={!available?.has_more} onClick={() => setPage(page + 1)}>
Next
</Button>
</div>
</>
);
}
function ConnectionMethod({
method,
setMethod,
label,
provider,
token,
setToken,
apiUrl,
setApiUrl,
connected,
apps,
busy,
connect,
}: {
method: "app" | "token";
setMethod: (value: "app" | "token") => void;
label: string;
provider: ObservedSettings["source_provider"];
token: string;
setToken: (value: string) => void;
apiUrl: string;
setApiUrl: (value: string) => void;
connected: ObservedConnection;
apps: z.infer<typeof appsSchema> | null;
busy: boolean;
connect: () => void;
}) {
const connectLabel = method === "app" ? `Connect ${label}` : "Continue";
const sameSource = connected.source_provider === provider && connected.api_url === apiUrl;
const hasSavedToken = sameSource && connected.has_token && connected.connection_type === "token";
return (
<>
<div className="grid grid-cols-2 gap-2">
<Button
variant={method === "app" ? "secondary" : "outline"}
aria-pressed={method === "app"}
onClick={() => setMethod("app")}
>
Connect with app
</Button>
<Button
variant={method === "token" ? "secondary" : "outline"}
aria-pressed={method === "token"}
onClick={() => setMethod("token")}
>
<KeyRound />
Access token
</Button>
</div>
{method === "token" && (
<TokenFields
label={label}
provider={provider}
token={token}
onToken={setToken}
apiUrl={apiUrl}
onUrl={setApiUrl}
hasToken={hasSavedToken}
/>
)}
{method === "app" && <AppMessage configured={Boolean(apps?.[provider].configured)} label={label} />}
<Button
className="w-full"
disabled={busy || (method === "app" && !apps?.[provider].configured)}
onClick={connect}
>
{busy ? "Connecting…" : connectLabel}
</Button>
</>
);
}
type ConnectionStep = "list" | "connect" | "repos";
const stepTitles = { list: "Connections", connect: "Connect your code", repos: "Choose repositories" };
function initialStep(settings: ObservedSettings, afterAuthorization: boolean): ConnectionStep {
if (afterAuthorization) return "repos";
if (settings.connections?.length) return "list";
return settings.has_token || settings.ready ? "repos" : "connect";
}
function stepDescription(step: ConnectionStep, label: string) {
if (step === "list") return "All selected repositories appear in one report";
if (step === "connect") return "Connect GitHub and GitLab with an app or access token";
return `Select ${label} repositories to compare`;
}
function connectionMethodLabel(entry: ObservedConnection) {
if (entry.connection_type === "app") return "App";
return entry.has_token ? "Token" : "Public access";
}
function ConnectionList({
connections,
onEdit,
onAdd,
}: {
connections: ObservedConnection[];
onEdit: (entry: ObservedConnection) => void;
onAdd: () => void;
}) {
return (
<>
{connections.map((entry) => (
<div key={entry.id} className="flex items-center justify-between gap-3 rounded-lg border p-4">
<div className="min-w-0 text-sm">
<p className="flex items-center gap-2 font-medium">
{entry.source_provider === "github" ? <Github className="size-4" /> : <Gitlab className="size-4" />}
{entry.source_provider === "github" ? "GitHub" : "GitLab"}
</p>
<p className="mt-1 truncate text-xs text-muted-foreground">{new URL(entry.api_url).host}</p>
<p className="mt-2 text-xs text-muted-foreground">
{entry.repos.length} repositories · {connectionMethodLabel(entry)}
</p>
</div>
<Button
variant="outline"
size="sm"
onClick={() => onEdit(entry)}
aria-label={`Edit ${entry.source_provider === "github" ? "GitHub" : "GitLab"} ${new URL(entry.api_url).host}`}
>
Edit
</Button>
</div>
))}
<Button className="w-full" variant="outline" onClick={onAdd}>
Add connection
</Button>
</>
);
}
function initialMethod(settings: ObservedSettings) {
return settings.has_token ? settings.connection_type : null;
}
function hasConnections(settings: ObservedSettings) {
return Boolean(settings.connections?.length);
}
function canManageApp(connected: ObservedConnection, apps: z.infer<typeof appsSchema> | null) {
return (
connected.connection_type === "app" && connected.source_provider === "github" && Boolean(apps?.github.can_install)
);
}
export default function ObservedConnections({
accessToken,
settings,
onClose,
onSaved,
initialError = "",
afterAuthorization = false,
}: {
accessToken: string;
settings: ObservedSettings;
onClose: () => void;
onSaved: () => void;
initialError?: string;
afterAuthorization?: boolean;
}) {
const [savedSettings, setSavedSettings] = useState(settings);
const [connected, setConnected] = useState<ObservedConnection>(settings);
const [provider, setProvider] = useState(settings.source_provider);
const [apiUrl, setApiUrl] = useState(settings.api_url);
const [selectedMethod, setMethod] = useState<"app" | "token" | null>(initialMethod(settings));
const [step, setStep] = useState<ConnectionStep>(() => initialStep(settings, afterAuthorization));
const [token, setToken] = useState("");
const [repos, setRepos] = useState(settings.repos.join(", "));
const [apps, setApps] = useState<z.infer<typeof appsSchema> | null>(null);
const method = preferredConnectionMethod(selectedMethod, apps?.[provider].configured);
const [available, setAvailable] = useState<z.infer<typeof repositoriesSchema> | null>(null);
const [query, setQuery] = useState("");
const [page, setPage] = useState(1);
const [busy, setBusy] = useState(false);
const [error, setError] = useState(initialError);
const label = provider === "github" ? "GitHub" : "GitLab";
const saveLabel = repositoryNames(repos).length ? "Save and sync" : "Save repositories";
const manageApp = canManageApp(connected, apps);
const showConnections = hasConnections(savedSettings);
useEffect(() => {
const controller = new AbortController();
apiClient
.get<unknown>("/roi-calculator/observed/apps", { accessToken, signal: controller.signal })
.then((data) => {
if (!controller.signal.aborted) setApps(appsSchema.parse(data));
})
.catch((reason: unknown) => {
if (!controller.signal.aborted) setError(extractProxyErrorMessage(reason));
});
return () => controller.abort();
}, [accessToken]);
useEffect(() => {
if (step !== "repos" || (!connected.has_token && connected.source_provider === "github")) return;
const controller = new AbortController();
const timer = setTimeout(() => {
apiClient
.get<unknown>("/roi-calculator/observed/repositories", {
accessToken,
signal: controller.signal,
query: { query, page, connection: connected.id },
})
.then((data) => {
if (!controller.signal.aborted) setAvailable(repositoriesSchema.parse(data));
})
.catch((reason: unknown) => {
if (!controller.signal.aborted) setError(extractProxyErrorMessage(reason));
});
}, 250);
return () => {
clearTimeout(timer);
controller.abort();
};
}, [accessToken, step, connected, query, page]);
function selectProvider(value: ObservedSettings["source_provider"]) {
setProvider(value);
const existing = savedSettings.connections?.find(
(entry) => entry.source_provider === value && entry.api_url === defaultUrl[value],
);
setApiUrl(existing?.api_url ?? defaultUrl[value]);
setConnected(
existing ?? {
...settings,
source_provider: value,
api_url: defaultUrl[value],
repos: [],
has_token: false,
ready: false,
connection_type: "token",
id: undefined,
},
);
setMethod(existing?.connection_type ?? null);
setToken("");
setError("");
}
function searchRepositories(value: string) {
setAvailable(null);
setError("");
setQuery(value);
}
function changePage(value: number) {
setAvailable(null);
setError("");
setPage(value);
}
async function connect(install = false) {
setBusy(true);
setError("");
try {
if (method === "app" || install) {
const sameApp = connected.source_provider === provider && connected.connection_type === "app";
const firstInstallation = provider === "github" && !sameApp && apps?.github.can_install;
const result = z.object({ url: z.string().url() }).parse(
await apiClient.post<unknown>(`/roi-calculator/observed/oauth/${provider}/start`, {
accessToken,
credentials: "include",
query: { install: install || Boolean(firstInstallation) },
}),
);
window.location.assign(result.url);
return;
}
const same = provider === connected.source_provider && apiUrl === connected.api_url;
const keepToken = same && connected.has_token && connected.connection_type === "token";
const result = observedSettingsSchema.parse(
await apiClient.put<unknown>("/roi-calculator/observed/settings", {
accessToken,
body: {
connection_id: savedSettings.connections?.find((entry) => entry.id === connected.id)?.id,
source_provider: provider,
api_url: apiUrl,
token: token || (keepToken ? undefined : ""),
repos: same ? connected.repos : [],
update_interval_minutes: connected.update_interval_minutes,
},
}),
);
setSavedSettings(result);
setConnected(result);
setToken("");
setRepos(result.repos.join(", "));
setAvailable(null);
setStep("repos");
} catch (reason) {
setError(extractProxyErrorMessage(reason));
} finally {
setBusy(false);
}
}
async function save() {
setBusy(true);
setError("");
try {
const result = observedSettingsSchema.parse(
await apiClient.put<unknown>("/roi-calculator/observed/settings", {
accessToken,
body: {
connection_id: connected.id,
source_provider: connected.source_provider,
api_url: connected.api_url,
repos: repositoryNames(repos),
update_interval_minutes: connected.update_interval_minutes,
},
}),
);
if (result.ready) await apiClient.post<unknown>("/roi-calculator/observed/sync", { accessToken });
onSaved();
onClose();
} catch (reason) {
setError(extractProxyErrorMessage(reason));
} finally {
setBusy(false);
}
}
function edit(entry: ObservedConnection) {
setConnected(entry);
setProvider(entry.source_provider);
setApiUrl(entry.api_url);
setMethod(entry.connection_type);
setRepos(entry.repos.join(", "));
setToken("");
setQuery("");
setPage(1);
setAvailable(null);
setError("");
setStep("repos");
}
return (
<Dialog
open
onOpenChange={(open) => {
if (!open) onClose();
}}
>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>{stepTitles[step]}</DialogTitle>
<DialogDescription>{stepDescription(step, label)}</DialogDescription>
</DialogHeader>
<div className="space-y-4 py-2">
{step === "list" && (
<ConnectionList
connections={savedSettings.connections ?? []}
onEdit={edit}
onAdd={() => {
selectProvider(
savedSettings.connections?.some((entry) => entry.source_provider === "github") ? "gitlab" : "github",
);
setStep("connect");
}}
/>
)}
{step === "connect" && (
<>
{showConnections && (
<Button variant="ghost" size="sm" onClick={() => setStep("list")}>
<ArrowLeft />
All connections
</Button>
)}
<div className="grid grid-cols-2 gap-3" role="group" aria-label="Code provider">
{(["github", "gitlab"] as const).map((value) => (
<Button
key={value}
variant={provider === value ? "default" : "outline"}
aria-pressed={provider === value}
onClick={() => selectProvider(value)}
>
{value === "github" ? <Github /> : <Gitlab />}
{value === "github" ? "GitHub" : "GitLab"}
</Button>
))}
</div>
<ConnectionMethod
method={method}
setMethod={setMethod}
label={label}
provider={provider}
token={token}
setToken={setToken}
apiUrl={apiUrl}
setApiUrl={setApiUrl}
connected={connected}
apps={apps}
busy={busy}
connect={() => connect()}
/>
</>
)}
{step === "repos" && (
<>
{showConnections && (
<Button variant="ghost" size="sm" onClick={() => setStep("list")}>
<ArrowLeft />
All connections
</Button>
)}
<Button variant="ghost" size="sm" onClick={() => setStep("connect")}>
<ArrowLeft />
Change connection
</Button>
{manageApp && (
<Button variant="outline" size="sm" disabled={busy} onClick={() => connect(true)}>
Manage GitHub repositories
</Button>
)}
<label htmlFor="roi-repositories" className="block text-sm font-medium">
Repositories
</label>
<Input
id="roi-repositories"
value={repos}
onChange={(event) => setRepos(event.target.value)}
placeholder={
provider === "github" ? "owner/repo, owner/another-repo" : "group/project, group/subgroup/project"
}
/>
{(connected.has_token || connected.source_provider === "gitlab") && (
<>
<RepositoryChoices
available={available}
repos={repos}
setRepos={setRepos}
query={query}
setQuery={searchRepositories}
page={page}
setPage={changePage}
/>
</>
)}
<Button className="w-full" disabled={busy} onClick={save}>
{busy ? "Saving…" : saveLabel}
</Button>
</>
)}
{error && (
<p role="alert" className="text-sm text-destructive">
{error}
</p>
)}
</div>
</DialogContent>
</Dialog>
);
}

232
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedDetails.tsx Normal file
View file

@ -0,0 +1,232 @@
"use client";
import { useState } from "react";
import { ExternalLink, GitPullRequest } from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetDescription } from "@/components/ui/sheet";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import {
dateRange,
changeTerms,
duration,
money,
number,
recordedBranches,
type Comparison,
type ObservedPerson,
type ObservedPull,
type ObservedSnapshot,
type Period,
} from "./observedData";
export function PullList({
pulls,
provider,
}: {
pulls: ObservedPull[];
provider: ObservedSnapshot["source_provider"];
}) {
const terms = changeTerms(provider);
const [query, setQuery] = useState("");
const [limit, setLimit] = useState(20);
const filtered = pulls.filter((pull) =>
`${pull.number} ${pull.title} ${pull.author} ${pull.repo} ${pull.source_repo} ${pull.source_branch}`
.toLowerCase()
.includes(query.toLowerCase()),
);
return (
<div className="space-y-3">
<Input
aria-label={`Search ${terms.lower}`}
placeholder={`Search ${terms.lower}…`}
value={query}
onChange={(event) => {
setQuery(event.target.value);
setLimit(20);
}}
className="max-w-sm"
/>
<Table>
<TableHeader>
<TableRow>
<TableHead>{terms.requests}</TableHead>
<TableHead>Author</TableHead>
<TableHead className="text-right">Opened to merged</TableHead>
<TableHead className="text-right">Tagged spend</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{filtered.slice(0, limit).map((pull) => (
<TableRow key={pull.url}>
<TableCell className="max-w-lg whitespace-normal py-3">
<a
className="group flex items-start gap-2 hover:underline"
href={pull.url}
target="_blank"
rel="noreferrer"
>
<GitPullRequest className="mt-0.5 size-4 shrink-0 text-violet-500" />
<span>
<span className="mr-2 text-muted-foreground">#{pull.number}</span>
{pull.title}
<span className="mt-1 block text-xs text-muted-foreground">{pull.repo}</span>
</span>
<ExternalLink className="mt-0.5 size-3 shrink-0 text-muted-foreground" />
</a>
</TableCell>
<TableCell>
<span className="text-xs">{pull.author || "Deleted author"}</span>
{pull.agent && (
<Badge variant="secondary" className="ml-2">
Agent
</Badge>
)}
</TableCell>
<TableCell className="text-right tabular-nums">{duration(pull.merge_hours)}</TableCell>
<TableCell
className="text-right tabular-nums"
title={
pull.branch_cost.status === "ambiguous"
? "Branch is shared by multiple changes"
: "Spend tagged to this branch during the period"
}
>
{money(pull.branch_cost.spend)}
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
<p className="text-xs text-muted-foreground">
Elapsed time from opening to merge, not engineering effort or time saved
</p>
{filtered.length === 0 && (
<p className="py-8 text-center text-sm text-muted-foreground">
{query ? `No ${terms.lower} match this search` : `No ${terms.lower} in this period`}
</p>
)}
<div className="flex items-center justify-between text-xs text-muted-foreground">
<span>
{number(Math.min(limit, filtered.length))} of {number(filtered.length)} {terms.lower}
</span>
{limit < filtered.length && (
<Button variant="outline" size="sm" onClick={() => setLimit(limit + 40)}>
Show more
</Button>
)}
</div>
</div>
);
}
export function PersonDetails({
person,
snapshot,
comparison,
onClose,
onEdit,
}: {
person: ObservedPerson;
snapshot: ObservedSnapshot;
comparison: Comparison;
onClose: () => void;
onEdit?: () => void;
}) {
const terms = changeTerms(snapshot.source_provider);
const [period, setPeriod] = useState<Period>("current");
const current = person.periods.current;
const baseline = person.periods[comparison];
const urls = new Set(person.periods[period].pr_urls);
const pulls = snapshot.pulls[period].filter((pull) => urls.has(pull.url));
return (
<Sheet
open
onOpenChange={(open) => {
if (!open) onClose();
}}
>
<SheetContent className="overflow-y-auto p-6 data-[side=right]:w-full data-[side=right]:sm:max-w-3xl">
<SheetHeader className="p-0 pr-8">
<SheetTitle className="text-xl">{person.name}</SheetTitle>
<SheetDescription>
{person.email} · {person.logins.join(", ")}
</SheetDescription>
</SheetHeader>
{onEdit && (
<Button variant="outline" className="w-fit" onClick={onEdit}>
Edit linked accounts
</Button>
)}
<div className="mt-2 grid grid-cols-1 gap-y-4 divide-y rounded-lg border py-5 sm:grid-cols-3 sm:divide-x sm:divide-y-0">
<div className="px-4">
<p className="text-xs text-muted-foreground">Merged {terms.plural}</p>
<p className="mt-2 text-2xl font-semibold">{number(current.merged_prs)}</p>
<p className="mt-1 text-xs text-muted-foreground">{number(baseline.merged_prs)} in comparison</p>
</div>
<div className="px-4">
<p className="text-xs text-muted-foreground">Recorded spend</p>
<p className="mt-2 text-2xl font-semibold">
{money(current.spend_observation === "no_records" ? null : current.gateway_recorded_spend)}
</p>
<p className="mt-1 text-xs text-muted-foreground">Gateway only</p>
</div>
<div className="px-4">
<p className="text-xs text-muted-foreground">Spend / matched {terms.singular}</p>
<p className="mt-2 text-2xl font-semibold">{money(current.recorded_spend_per_attributed_pr)}</p>
<p className="mt-1 text-xs text-muted-foreground">Period average</p>
</div>
</div>
<div className="mt-3 flex flex-wrap items-center justify-between gap-3">
<h3 className="font-medium">Merged {terms.plural}</h3>
<div className="flex gap-1 rounded-lg bg-muted p-1">
<Button size="sm" variant={period === "current" ? "outline" : "ghost"} onClick={() => setPeriod("current")}>
Current
</Button>
<Button
size="sm"
variant={period === comparison ? "outline" : "ghost"}
onClick={() => setPeriod(comparison)}
>
{comparison === "previous" ? "Previous period" : "Last year"}
</Button>
</div>
</div>
<p className="text-xs text-muted-foreground">{dateRange(snapshot.periods[period].window)} · UTC</p>
<PullList key={period} pulls={pulls} provider={snapshot.source_provider} />
</SheetContent>
</Sheet>
);
}
export function BranchSpend({ snapshot }: { snapshot: ObservedSnapshot }) {
const rows = recordedBranches(snapshot);
return (
<div className="rounded-xl border">
<Table>
<TableHeader>
<TableRow>
<TableHead>Repository</TableHead>
<TableHead>Branch</TableHead>
<TableHead className="text-right">Requests</TableHead>
<TableHead className="text-right">Tagged spend</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{rows.map((row) => (
<TableRow key={`${row.repo}/${row.branch}`}>
<TableCell>{row.repo}</TableCell>
<TableCell>{row.branch}</TableCell>
<TableCell className="text-right">{number(row.requests)}</TableCell>
<TableCell className="text-right">{money(row.spend)}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
{rows.length === 0 && (
<p className="p-8 text-center text-sm text-muted-foreground">No tagged branch spend in this period</p>
)}
</div>
);
}

305
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.integration.test.tsx Normal file
View file

@ -0,0 +1,305 @@
import { fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, describe, expect, it, vi } from "vitest";
import ObservedROIView from "./ObservedROIView";
import type { ObservedSettings, ObservedSnapshot, ObservedStatus } from "./observedData";
const settings: ObservedSettings = {
source_provider: "gitlab",
api_url: "https://gitlab.com/api/v4",
repos: ["org/service"],
has_token: true,
connection_type: "app",
update_interval_minutes: 1440,
ready: true,
};
const idle: ObservedStatus = {
running: false,
phase: "complete",
stage: "",
done: 0,
total: 0,
error: null,
finished_at: null,
};
const period = {
window: { start: "2026-09-01", end: "2026-09-28" },
merged_prs: 1,
median_merge_hours: 16 / 3600,
human_authored: 1,
agent_authored: 0,
missing_author: 0,
agents_without_requester: 0,
matched_internal_prs: 0,
new_bug_labeled_issues: 0,
new_regression_labeled_issues: 0,
explicitly_titled_revert_prs: 0,
matched_users_recorded_spend: 0,
spend_observation: "no_records" as const,
human_summary: { median_merge_hours: 16 / 3600 },
};
const report: ObservedSnapshot = {
source_provider: "gitlab",
repos: ["org/service"],
unmatched_logins: [],
unlinked_branches: [],
captured_at: "2026-09-29T00:00:00Z",
periods: { current: period, previous: period, last_year: period },
people: [],
pulls: { current: [], previous: [], last_year: [] },
};
const app = { configured: true, api_url: null, callback_url: null };
afterEach(() => {
vi.unstubAllGlobals();
window.history.replaceState(null, "", "/");
});
describe("observed ROI dashboard", () => {
it("previews every sample view before setup, changes sample periods without writes, and exits back to setup", async () => {
const requests = vi.fn(async (input: string, _init: RequestInit) => {
const path = new URL(input, "http://localhost").pathname;
const disconnected = { ...settings, ready: false, repos: [], has_token: false };
if (path.endsWith("/settings")) return Response.json(disconnected);
if (path.endsWith("/report")) return Response.json({ report: null });
if (path.endsWith("/sync")) return Response.json(idle);
throw new Error(path);
});
vi.stubGlobal("fetch", requests);
const user = userEvent.setup();
render(<ObservedROIView accessToken="test-only-gateway-token" />);
expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeInTheDocument();
expect(screen.queryByText("Ready to sync")).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "Preview sample report" }));
expect(screen.getByRole("status")).toHaveTextContent("You’re viewing demo data");
expect(screen.getByRole("tab", { name: "Engineers 3", selected: true })).toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Connections" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Link accounts" })).not.toBeInTheDocument();
expect(window.location.search).toBe("?demo=1");
await user.click(screen.getByRole("button", { name: "View Alex Rivera's merged changes" }));
expect(await screen.findByRole("dialog", { name: "Alex Rivera" })).toHaveTextContent("alex-demo@example.com");
expect(screen.getByRole("heading", { name: "Merged changes" })).toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Edit linked accounts" })).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "Close" }));
await user.click(screen.getByRole("tab", { name: "Merged changes" }));
expect(screen.getByRole("img", { name: /Merged changes by week/ })).toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "Quality" }));
expect(screen.getByText("New regression-labeled issues")).toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "Branch spend" }));
expect(screen.getAllByText(/feature\/sample-/).length).toBeGreaterThan(0);
await user.click(screen.getByRole("combobox", { name: "Reporting period" }));
await user.click(await screen.findByRole("option", { name: "Last 7 days" }));
expect(screen.getByRole("combobox", { name: "Reporting period" })).toHaveTextContent("Last 7 days");
await user.click(screen.getByRole("combobox", { name: "Comparison period" }));
await user.click(await screen.findByRole("option", { name: "vs. same period last year" }));
expect(screen.getByRole("combobox", { name: "Comparison period" })).toHaveTextContent("vs. same period last year");
await user.click(screen.getByRole("button", { name: "Exit demo" }));
expect(screen.getByRole("heading", { name: "Connect your repositories" })).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Connect GitHub or GitLab" })).toBeEnabled();
expect(window.location.search).toBe("");
expect(requests.mock.calls.every(([, init]) => init.method === "GET")).toBe(true);
});
it("keeps live sync and its report intact when entering and exiting the demo", async () => {
const requests = vi.fn(async (input: string, _init: RequestInit) => {
const path = new URL(input, "http://localhost").pathname;
if (path.endsWith("/settings")) return Response.json(settings);
if (path.endsWith("/report")) return Response.json({ report });
if (path.endsWith("/sync")) return Response.json({ ...idle, running: true, stage: "Reading changes" });
throw new Error(path);
});
vi.stubGlobal("fetch", requests);
window.history.replaceState(null, "", "/roi-calculator/?from=review#report");
const user = userEvent.setup();
render(<ObservedROIView accessToken="test-only-gateway-token" />);
expect(await screen.findByRole("button", { name: "Cancel sync" })).toBeEnabled();
await user.click(screen.getByRole("button", { name: "Preview sample report" }));
expect(screen.queryByRole("button", { name: "Cancel sync" })).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: "2 repositories" })).toBeInTheDocument();
expect(window.location.search).toBe("?from=review&demo=1");
await user.click(screen.getByRole("button", { name: "Exit demo" }));
expect(screen.getByRole("button", { name: "Cancel sync" })).toBeEnabled();
expect(screen.getByRole("button", { name: "1 repository" })).toBeInTheDocument();
expect(screen.getByRole("tab", { name: "Merge requests", selected: true })).toBeInTheDocument();
expect(window.location.search).toBe("?from=review");
expect(window.location.hash).toBe("#report");
expect(requests.mock.calls.every(([, init]) => init.method === "GET")).toBe(true);
});
it.each(["failed", "pending"])("opens a demo URL even when live requests are %s", async (state) => {
window.history.replaceState(null, "", "/roi-calculator/?demo=1");
const pending = Promise.withResolvers<Response>();
vi.stubGlobal(
"fetch",
vi.fn(() => (state === "failed" ? Promise.reject(new Error("Live data unavailable")) : pending.promise)),
);
const user = userEvent.setup();
render(<ObservedROIView accessToken="test-only-gateway-token" isViewOnly />);
expect(screen.getByRole("status")).toHaveTextContent("You’re viewing demo data");
expect(screen.getByText("Alex Rivera")).toBeInTheDocument();
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "Exit demo" }));
expect(screen.queryByText("Alex Rivera")).not.toBeInTheDocument();
expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument();
expect(window.location.search).toBe("");
if (state === "failed") expect(await screen.findByRole("alert")).toHaveTextContent("Live data unavailable");
});
it("retries failures, keeps the report during cancellation, and refreshes after completion", async () => {
let status: ObservedStatus = { ...idle, phase: "error", error: "Provider temporarily unavailable" };
let completeOnPoll = false;
let currentReport = { ...report, repos: ["org/service", "org/docs"] };
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
const path = new URL(input, "http://localhost").pathname;
if (path.endsWith("/settings")) return Response.json(settings);
if (path.endsWith("/report")) return Response.json({ report: currentReport });
if (path.endsWith("/sync")) {
if (init.method === "POST") status = { ...idle, running: true, phase: "pulls" };
if (init.method === "DELETE") status = { ...idle, phase: "cancelled" };
if (init.method === "GET" && completeOnPoll) {
status = { ...idle, finished_at: "2026-09-29T00:01:00Z" };
currentReport = { ...report, repos: ["org/updated"] };
}
return Response.json(status);
}
throw new Error(path);
}),
);
const user = userEvent.setup();
render(<ObservedROIView accessToken="test-only-gateway-token" />);
expect(await screen.findByRole("tab", { name: "Merge requests", selected: true })).toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "Quality" }));
expect(screen.getByRole("alert")).toHaveTextContent("Provider temporarily unavailable");
await user.click(screen.getByRole("button", { name: "Retry" }));
await user.click(await screen.findByRole("button", { name: "Cancel sync" }));
expect(await screen.findByRole("button", { name: "Sync now" })).toBeEnabled();
expect(screen.queryByText("org/service")).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "2 repositories" }));
const repositories = await screen.findByRole("dialog", { name: "Repositories" });
expect(within(repositories).getByText("org/service")).toBeInTheDocument();
expect(within(repositories).getByText("org/docs")).toBeInTheDocument();
await user.keyboard("{Escape}");
await waitFor(() => expect(screen.queryByRole("dialog", { name: "Repositories" })).not.toBeInTheDocument());
await user.click(screen.getByRole("button", { name: "Sync now" }));
expect(await screen.findByRole("button", { name: "Cancel sync" })).toBeEnabled();
completeOnPoll = true;
await user.click(await screen.findByRole("button", { name: "1 repository" }, { timeout: 4000 }));
expect(
await within(screen.getByRole("dialog", { name: "Repositories" })).findByText("org/updated"),
).toBeInTheDocument();
await user.keyboard("{Escape}");
expect(screen.getByRole("tab", { name: "Quality", selected: true })).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Sync now" })).toBeEnabled();
});
it("syncs the selected range and labels its equal-length comparison", async () => {
let currentReport = report;
const requested: string[] = [];
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
const url = new URL(input, "http://localhost");
if (url.pathname.endsWith("/settings")) return Response.json(settings);
if (url.pathname.endsWith("/report")) return Response.json({ report: currentReport });
if (url.pathname.endsWith("/sync")) {
if (init.method === "POST") {
requested.push(url.searchParams.get("days") ?? "");
currentReport = {
...report,
periods: {
...report.periods,
current: { ...period, window: { start: "2026-09-22", end: "2026-09-28" } },
previous: { ...period, window: { start: "2026-09-15", end: "2026-09-21" } },
},
};
}
return Response.json(idle);
}
throw new Error(url.pathname);
}),
);
const user = userEvent.setup();
render(<ObservedROIView accessToken="test-only-gateway-token" />);
await user.click(await screen.findByRole("combobox", { name: "Reporting period" }));
await user.click(await screen.findByRole("option", { name: "Last 7 days" }));
await waitFor(() => expect(requested).toEqual(["7"]));
expect(await screen.findByText(/Comparing with Sep 15.*Sep 21/)).toBeInTheDocument();
expect(screen.getByRole("combobox", { name: "Reporting period" })).toHaveTextContent("Last 7 days");
expect(screen.getByRole("combobox", { name: "Comparison period" })).toHaveTextContent("vs. previous period");
});
it("shows a successful empty repository without a setup prompt or invented durations", async () => {
const emptyPeriod = {
...period,
merged_prs: 0,
human_authored: 0,
median_merge_hours: null,
human_summary: { median_merge_hours: null },
};
const empty = { ...report, periods: { current: emptyPeriod, previous: emptyPeriod, last_year: emptyPeriod } };
vi.stubGlobal(
"fetch",
vi.fn(async (input: string) => {
const path = new URL(input, "http://localhost").pathname;
if (path.endsWith("/settings")) return Response.json(settings);
if (path.endsWith("/report")) return Response.json({ report: empty });
if (path.endsWith("/sync")) return Response.json(idle);
throw new Error(path);
}),
);
render(<ObservedROIView accessToken="test-only-gateway-token" />);
expect(await screen.findByRole("heading", { name: "No merged changes yet" })).toBeInTheDocument();
expect(screen.getByText("No merges")).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Sync now" })).toBeEnabled();
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument();
expect(screen.queryByText("0h")).not.toBeInTheDocument();
});
it.each([
{ query: "connected=gitlab", alerts: [] },
{ query: "connection_failed=1", alerts: ["Connection failed or expired. Try again or use a token"] },
{ query: "connection_cancelled=1", alerts: ["Connection cancelled. Choose an app or token to try again"] },
])("resumes setup after $query and refreshes saved changes after closing", async ({ query, alerts }) => {
window.history.replaceState(null, "", `/roi-calculator/?${query}`);
let currentSettings = settings;
vi.stubGlobal(
"fetch",
vi.fn(async (input: string, init: RequestInit) => {
const path = new URL(input, "http://localhost").pathname;
if (path.endsWith("/apps")) return Response.json({ github: app, gitlab: app });
if (path.endsWith("/repositories")) return Response.json({ repositories: [], has_more: false });
if (path.endsWith("/settings")) {
if (init.method === "PUT") currentSettings = { ...settings, repos: ["org/changed"] };
return Response.json(currentSettings);
}
if (path.endsWith("/report")) return Response.json({ report: { ...report, repos: currentSettings.repos } });
if (path.endsWith("/sync"))
return init.method === "POST"
? Response.json({ detail: "Provider unavailable" }, { status: 502 })
: Response.json(idle);
throw new Error(path);
}),
);
const user = userEvent.setup();
render(<ObservedROIView accessToken="test-only-gateway-token" />);
const dialog = await screen.findByRole("dialog", { name: "Choose repositories" });
expect(
within(dialog)
.queryAllByRole("alert")
.map((alert) => alert.textContent),
).toEqual(alerts);
expect(window.location.search).toBe("");
fireEvent.change(within(dialog).getByLabelText("Repositories"), { target: { value: "org/changed" } });
await user.click(within(dialog).getByRole("button", { name: "Save and sync" }));
expect(await within(dialog).findByRole("alert")).toHaveTextContent("Provider unavailable");
await user.click(within(dialog).getByRole("button", { name: "Close" }));
await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument());
await user.click(await screen.findByRole("button", { name: "1 repository" }));
expect(
await within(screen.getByRole("dialog", { name: "Repositories" })).findByText("org/changed"),
).toBeInTheDocument();
});
});

253
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.tsx Normal file
View file

@ -0,0 +1,253 @@
"use client";
import { useEffect, useState } from "react";
import { Link2, RefreshCw } from "lucide-react";
import { apiClient } from "@/components/networking";
import { extractProxyErrorMessage } from "@/lib/http/client";
import { Page } from "@/components/shared/Page";
import { PageHeader, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader";
import { DemoNotice } from "@/components/shared/DemoNotice";
import { Button } from "@/components/ui/button";
import { Skeleton } from "@/components/ui/skeleton";
import ObservedConnections from "./ObservedConnections";
import ObservedReport from "./ObservedReport";
import { useObservedReport, type ObservedViewData } from "./useObservedReport";
import { syncMessage, type ObservedSnapshot } from "./observedData";
import { createObservedDemo } from "./observedDemo";
function SyncActions({
data,
error,
busy,
readOnly,
compact = false,
onSync,
onRetry,
}: {
data: ObservedViewData | null;
error: string;
busy: boolean;
readOnly: boolean;
compact?: boolean;
onSync: (cancel: boolean) => void;
onRetry: () => void;
}) {
const message = error || data?.status.error;
const statusMessage = data ? syncMessage(data.status, data.report) : "";
const canSync = data?.settings.ready && !readOnly;
if (!message && !statusMessage && !canSync) return null;
return (
<div className={compact ? "contents" : "space-y-2"}>
{message && (
<div
role="alert"
className="flex basis-full items-center justify-between gap-3 rounded-lg border border-destructive/30 p-3 text-sm text-destructive"
>
<span>{message}</span>
<Button size="sm" variant="outline" onClick={onRetry}>
Retry
</Button>
</div>
)}
{data && (
<div className={compact ? "contents" : "flex flex-wrap items-center justify-between gap-2"}>
{statusMessage && (
<span role="status" className="text-xs text-muted-foreground">
{statusMessage}
</span>
)}
{!readOnly && data.settings.ready && (
<Button size="sm" variant="outline" disabled={busy} onClick={() => onSync(data.status.running)}>
<RefreshCw className={data.status.running ? "animate-spin" : ""} />
{data.status.running ? "Cancel sync" : "Sync now"}
</Button>
)}
</div>
)}
</div>
);
}
function EmptyReport({
data,
readOnly,
onConnect,
}: {
data: ObservedViewData;
readOnly: boolean;
onConnect: () => void;
}) {
function title() {
if (data.status.running) return "Reading repository activity";
return data.settings.ready ? "Ready for your first report" : "Connect your repositories";
}
return (
<div className="flex flex-col items-center gap-4 px-6 py-12 text-center">
<h2 className="text-lg font-medium">{title()}</h2>
<p className="max-w-md text-sm text-muted-foreground">
{data.status.running
? "Your report will appear here when the first sync finishes"
: "Compare merged changes, issue trends, and recorded AI spend across your team"}
</p>
{!readOnly && !data.status.running && (
<Button onClick={onConnect}>
<Link2 />
{data.settings.ready ? "Connections" : "Connect GitHub or GitLab"}
</Button>
)}
</div>
);
}
export default function ObservedROIView({
accessToken,
isViewOnly = false,
}: {
accessToken: string;
isViewOnly?: boolean;
}) {
const { data, error, refresh } = useObservedReport(accessToken);
const [returned] = useState(() => new URLSearchParams(typeof window === "undefined" ? "" : window.location.search));
const [sample, setSample] = useState<ObservedSnapshot | null>(() =>
returned.get("demo") === "1" ? createObservedDemo(28) : null,
);
const [connections, setConnections] = useState(
["github", "gitlab"].includes(returned.get("connected") ?? "") ||
returned.has("connection_cancelled") ||
returned.has("connection_failed"),
);
const [connectionError, setConnectionError] = useState(() => {
if (returned.has("connection_failed")) return "Connection failed or expired. Try again or use a token";
if (returned.has("connection_cancelled")) return "Connection cancelled. Choose an app or token to try again";
return "";
});
const [afterAuthorization, setAfterAuthorization] = useState(Boolean(returned.get("connected")));
const [busy, setBusy] = useState(false);
const [actionError, setActionError] = useState("");
useEffect(() => {
const url = new URL(window.location.href);
url.searchParams.delete("connected");
url.searchParams.delete("connection_cancelled");
url.searchParams.delete("connection_failed");
window.history.replaceState(window.history.state, "", url);
}, []);
function previewSample(enabled: boolean) {
const url = new URL(window.location.href);
if (enabled) url.searchParams.set("demo", "1");
else url.searchParams.delete("demo");
window.history.replaceState(window.history.state, "", url);
setSample(enabled ? createObservedDemo(28) : null);
}
function closeConnections() {
setConnections(false);
setConnectionError("");
setAfterAuthorization(false);
refresh();
}
async function sync(cancel: boolean, days?: number) {
setBusy(true);
setActionError("");
try {
if (cancel) await apiClient.delete("/roi-calculator/observed/sync", { accessToken });
else await apiClient.post("/roi-calculator/observed/sync", { accessToken, query: { days } });
refresh();
} catch (reason) {
setActionError(extractProxyErrorMessage(reason));
} finally {
setBusy(false);
}
}
function retry() {
if (error || isViewOnly || !data?.settings.ready) refresh();
else void sync(data.status.running);
}
if (sample) {
return (
<ObservedReport
key="demo"
snapshot={sample}
accessToken={accessToken}
readOnly
onRefresh={refresh}
onConnect={() => setConnections(true)}
actions={null}
notice={<DemoNotice onExit={() => previewSample(false)} />}
syncing={false}
onPeriod={(days) => setSample(createObservedDemo(days))}
/>
);
}
const previewButton = (
<Button size="sm" variant="ghost" onClick={() => previewSample(true)}>
Preview sample report
</Button>
);
const actions = (
<>
{previewButton}
<SyncActions
data={data}
error={error || actionError}
busy={busy}
readOnly={isViewOnly}
compact={Boolean(data?.report)}
onSync={sync}
onRetry={retry}
/>
</>
);
const content = data?.report ? (
<ObservedReport
key="live"
snapshot={data.report}
accessToken={accessToken}
readOnly={isViewOnly}
onRefresh={refresh}
onConnect={() => setConnections(true)}
actions={actions}
syncing={busy || data.status.running}
onPeriod={isViewOnly ? undefined : (days) => void sync(false, days)}
/>
) : (
<Page>
<PageHeader>
<div className="flex flex-wrap items-center justify-between gap-3">
<PageHeaderTitle>ROI Calculator</PageHeaderTitle>
{previewButton}
</div>
<PageHeaderDescription>Are we shipping more, with fewer bugs, at a better cost?</PageHeaderDescription>
</PageHeader>
<SyncActions
data={data}
error={error || actionError}
busy={busy}
readOnly={isViewOnly}
onSync={sync}
onRetry={retry}
/>
{!data && !error && (
<>
<Skeleton className="h-32 w-full" />
<Skeleton className="h-96 w-full" />
</>
)}
{data && <EmptyReport data={data} readOnly={isViewOnly} onConnect={() => setConnections(true)} />}
</Page>
);
const showConnections = connections && data && !isViewOnly;
return (
<>
{content}
{showConnections && (
<ObservedConnections
accessToken={accessToken}
settings={data.settings}
initialError={connectionError}
afterAuthorization={afterAuthorization}
onClose={closeConnections}
onSaved={refresh}
/>
)}
</>
);
}

611
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedReport.tsx Normal file
View file

@ -0,0 +1,611 @@
"use client";
import { useState } from "react";
import { ArrowDown, ArrowUp, CalendarDays, ChevronDown, ChevronRight, Link2, Search, Users } from "lucide-react";
import { Page, PageTabsList, PageTabsTrigger } from "@/components/shared/Page";
import { PageHeader, PageHeaderTitle } from "@/components/shared/PageHeader";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Tabs, TabsContent } from "@/components/ui/tabs";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import ObservedAccounts from "./ObservedAccounts";
import { BranchSpend, PersonDetails, PullList } from "./ObservedDetails";
import {
change,
dateRange,
changeTerms,
duration,
money,
number,
visiblePeople,
weeklyMerges,
type Comparison,
type ObservedPerson,
type ObservedSnapshot,
type PeopleSort,
} from "./observedData";
function Delta({
current,
baseline,
neutral = false,
}: {
current: number | null;
baseline: number | null;
neutral?: boolean;
}) {
const delta = current === null || baseline === null ? null : change(current, baseline);
if (delta === null) return <span className="text-xs text-muted-foreground">No baseline</span>;
const Icon = delta >= 0 ? ArrowUp : ArrowDown;
return (
<span
aria-label={`${number(Math.abs(delta))}% ${delta >= 0 ? "increase" : "decrease"}`}
className={`inline-flex items-center gap-1 text-xs tabular-nums ${neutral ? "text-muted-foreground" : "text-foreground"}`}
>
<Icon className="size-3" />
{number(Math.abs(delta))}%
</span>
);
}
function Metric({
label,
value,
detail,
current,
baseline,
}: {
label: string;
value: string;
detail: string;
current?: number | null;
baseline?: number | null;
}) {
return (
<div className="min-w-0 bg-background px-4 py-3">
<div className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">{label}</div>
<div className="mt-1 flex flex-wrap items-baseline gap-2">
<span className="text-2xl font-semibold tracking-tight tabular-nums">{value}</span>
{current !== undefined && baseline !== undefined && <Delta current={current} baseline={baseline} />}
</div>
<p className="mt-1 text-xs text-muted-foreground">{detail}</p>
</div>
);
}
function ShippingTrend({ snapshot, comparison }: { snapshot: ObservedSnapshot; comparison: Comparison }) {
const terms = changeTerms(snapshot.source_provider);
const current = weeklyMerges(snapshot, "current");
const baseline = weeklyMerges(snapshot, comparison);
const max = Math.max(1, ...current, ...baseline);
return (
<div className="rounded-xl border p-5">
<div className="flex flex-wrap items-center justify-between gap-2">
<h2 className="text-sm font-medium">Shipping activity</h2>
<div className="flex gap-4 text-xs text-muted-foreground">
<span className="flex items-center gap-1.5">
<span className="size-2 rounded-sm bg-blue-500" />
Current period
</span>
<span className="flex items-center gap-1.5">
<span className="size-2 rounded-sm bg-slate-300 dark:bg-slate-600" />
{comparison === "previous" ? "Previous period" : "Last year"}
</span>
</div>
</div>
<div
className="mt-5 grid gap-2"
style={{ gridTemplateColumns: `repeat(${current.length}, minmax(0, 1fr))` }}
role="img"
aria-label={`Merged ${terms.plural} by week. Current: ${current.join(", ")}. Comparison: ${baseline.join(", ")}`}
>
{current.map((value, week) => (
<div key={week} className="min-w-0">
<div className="flex h-28 items-end justify-center gap-2 border-b border-border/60">
<div
className="group relative w-10 rounded-t bg-slate-200 dark:bg-slate-700"
style={{ height: `${Math.max(2, (baseline[week] / max) * 85)}%` }}
>
<span className="absolute -top-5 left-1/2 -translate-x-1/2 text-[10px] text-muted-foreground">
{baseline[week]}
</span>
</div>
<div
className="relative w-10 rounded-t bg-blue-500/90"
style={{ height: `${Math.max(2, (value / max) * 85)}%` }}
>
<span className="absolute -top-5 left-1/2 -translate-x-1/2 text-[10px] font-medium">{value}</span>
</div>
</div>
<p className="mt-2 text-center text-[11px] text-muted-foreground">W{week + 1}</p>
</div>
))}
</div>
</div>
);
}
function PeopleTable({
snapshot,
comparison,
onSelect,
}: {
snapshot: ObservedSnapshot;
comparison: Comparison;
onSelect: (person: ObservedPerson) => void;
}) {
const terms = changeTerms(snapshot.source_provider);
const [query, setQuery] = useState("");
const [sort, setSort] = useState<PeopleSort>("merged");
const people = visiblePeople(snapshot.people, query, sort);
return (
<div className="space-y-3">
<div className="flex flex-wrap items-center justify-between gap-3">
<div className="relative w-72">
<Search className="absolute top-2.5 left-3 size-4 text-muted-foreground" />
<Input
aria-label="Search engineers"
placeholder="Search engineers…"
value={query}
onChange={(event) => setQuery(event.target.value)}
className="pl-9"
/>
</div>
<div className="flex items-center gap-3">
<span className="text-xs text-muted-foreground">
{people.length} {people.length === 1 ? "engineer" : "engineers"}
</span>
<Select
value={sort}
onValueChange={(value) => {
if (value) setSort(value);
}}
items={[
{ value: "merged", label: `Most merged ${terms.plural}` },
{ value: "spend", label: "Highest spend" },
{ value: "cost", label: `Highest cost / ${terms.singular}` },
{ value: "name", label: "Name" },
]}
>
<SelectTrigger aria-label="Sort engineers">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="merged">Most merged {terms.plural}</SelectItem>
<SelectItem value="spend">Highest spend</SelectItem>
<SelectItem value="cost">Highest cost / {terms.singular}</SelectItem>
<SelectItem value="name">Name</SelectItem>
</SelectContent>
</Select>
</div>
</div>
<div className="overflow-hidden rounded-xl border">
<Table>
<TableHeader>
<TableRow className="bg-muted/30">
<TableHead className="pl-5">Engineer</TableHead>
<TableHead className="text-right">Merged {terms.plural}</TableHead>
<TableHead>Authored / agent</TableHead>
<TableHead className="text-right">
{comparison === "previous" ? "vs. previous" : "vs. last year"}
</TableHead>
<TableHead className="text-right">Median merge</TableHead>
<TableHead className="text-right">Recorded spend</TableHead>
<TableHead className="text-right">Spend / {terms.singular}</TableHead>
<TableHead>
<span className="sr-only">Details</span>
</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{people.map((person) => {
const current = person.periods.current;
const baseline = person.periods[comparison];
return (
<TableRow key={person.email}>
<TableCell className="py-3 pl-5">
<button
className="flex items-center gap-3 rounded text-left hover:underline"
onClick={() => onSelect(person)}
>
<span className="flex size-8 shrink-0 items-center justify-center rounded-full bg-muted text-xs font-medium text-muted-foreground">
{person.name.slice(0, 2).toUpperCase()}
</span>
<span>
<span className="block font-medium">{person.name}</span>
<span className="text-xs text-muted-foreground">{person.email}</span>
</span>
</button>
</TableCell>
<TableCell className="text-right font-medium tabular-nums">{number(current.merged_prs)}</TableCell>
<TableCell>
<div className="flex w-28 items-center gap-2">
<div className="flex h-1.5 w-16 overflow-hidden rounded-full bg-muted">
<span
className="bg-blue-500"
style={{
width: `${current.merged_prs ? (current.direct_authored / current.merged_prs) * 100 : 0}%`,
}}
/>
<span
className="bg-violet-400"
style={{
width: `${current.merged_prs ? (current.declared_agent_owned / current.merged_prs) * 100 : 0}%`,
}}
/>
</div>
<span className="text-[11px] text-muted-foreground">
{current.direct_authored}/{current.declared_agent_owned}
</span>
</div>
</TableCell>
<TableCell className="text-right">
<Delta current={current.merged_prs} baseline={baseline.merged_prs} neutral />
<span className="ml-2 text-xs text-muted-foreground">({baseline.merged_prs})</span>
</TableCell>
<TableCell className="text-right tabular-nums">{duration(current.median_merge_hours)}</TableCell>
<TableCell className="text-right tabular-nums">
{money(current.spend_observation === "no_records" ? null : current.gateway_recorded_spend)}
</TableCell>
<TableCell className="text-right font-medium tabular-nums">
{money(current.recorded_spend_per_attributed_pr)}
</TableCell>
<TableCell>
<Button
size="icon-xs"
variant="ghost"
aria-label={`View ${person.name}'s ${terms.lower}`}
onClick={() => onSelect(person)}
>
<ChevronRight />
</Button>
</TableCell>
</TableRow>
);
})}
</TableBody>
</Table>
{people.length === 0 && (
<div className="p-10 text-center text-sm text-muted-foreground">
{query ? `No engineers match “${query}”` : "Link accounts to see your engineers"}
</div>
)}
</div>
<p className="text-xs text-muted-foreground">
<span className="mr-3 inline-flex items-center gap-1.5">
<span className="size-2 rounded-sm bg-blue-500" />
Authored
</span>
<span className="mr-3 inline-flex items-center gap-1.5">
<span className="size-2 rounded-sm bg-violet-400" />
Agent, explicit requester
</span>
Spend / {terms.singular} is recorded period spend divided by matched {terms.plural}
</p>
</div>
);
}
function Quality({ snapshot, comparison }: { snapshot: ObservedSnapshot; comparison: Comparison }) {
const terms = changeTerms(snapshot.source_provider);
const current = snapshot.periods.current;
const baseline = snapshot.periods[comparison];
const rows = [
{
label: "New bug-labeled issues",
current: current.new_bug_labeled_issues,
baseline: baseline.new_bug_labeled_issues,
detail: "Opened during the period, with bug or kind:bug labels at collection",
},
{
label: "New regression-labeled issues",
current: current.new_regression_labeled_issues,
baseline: baseline.new_regression_labeled_issues,
detail: "Opened during the period and labeled as regressions",
},
{
label: `Revert-titled ${terms.plural}`,
current: current.explicitly_titled_revert_prs,
baseline: baseline.explicitly_titled_revert_prs,
detail: `Merged ${terms.plural} whose titles explicitly indicate a revert`,
},
];
return (
<div className="space-y-4">
<div className="rounded-xl border">
<Table>
<TableHeader>
<TableRow>
<TableHead className="pl-5">Repository signal</TableHead>
<TableHead className="text-right">Current</TableHead>
<TableHead className="text-right">Comparison</TableHead>
<TableHead className="pr-5 text-right">Change</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{rows.map((row) => (
<TableRow key={row.label}>
<TableCell className="py-5 pl-5">
<p className="font-medium">{row.label}</p>
<p className="mt-1 text-xs text-muted-foreground">{row.detail}</p>
</TableCell>
<TableCell className="text-right font-medium">{number(row.current)}</TableCell>
<TableCell className="text-right text-muted-foreground">{number(row.baseline)}</TableCell>
<TableCell className="pr-5 text-right">
<Delta current={row.current} baseline={row.baseline} neutral />
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
<p className="max-w-3xl text-xs leading-relaxed text-muted-foreground">
These signals help check whether more shipping comes with more bugs. Labels and revert titles are incomplete
proxies; they do not establish a change-failure rate or attribute bugs to an engineer.
</p>
</div>
);
}
function costPerChange(period: ObservedSnapshot["periods"]["current"]) {
if (period.spend_observation !== "records_present" || period.matched_internal_prs === 0) return null;
return period.matched_users_recorded_spend / period.matched_internal_prs;
}
export default function ObservedReport({
snapshot,
accessToken,
readOnly,
onRefresh,
onConnect,
actions,
notice,
syncing,
onPeriod,
}: {
snapshot: ObservedSnapshot;
accessToken: string;
readOnly: boolean;
onRefresh: () => void;
onConnect: () => void;
actions: React.ReactNode;
notice?: React.ReactNode;
syncing: boolean;
onPeriod?: (days: number) => void;
}) {
const [comparison, setComparison] = useState<Comparison>("previous");
const [activeTab, setActiveTab] = useState(
snapshot.people.length && snapshot.periods.current.merged_prs > 0 ? "people" : "pulls",
);
const [accountEmail, setAccountEmail] = useState<string | null>(null);
const [personEmail, setPersonEmail] = useState<string | null>(null);
const person = snapshot.people.find((entry) => entry.email === personEmail) ?? null;
const terms = changeTerms(snapshot.source_provider);
const current = snapshot.periods.current;
const baseline = snapshot.periods[comparison];
const days = Math.round((Date.parse(current.window.end) - Date.parse(current.window.start)) / 86400000) + 1;
const rangeOptions = [...new Set([7, 28, 90, days])].sort((a, b) => a - b);
const cost = costPerChange(current);
const baselineCost = costPerChange(baseline);
return (
<Page className="mx-auto max-w-[1500px] gap-3 pb-10 sm:pt-4">
<PageHeader className="flex flex-wrap items-center justify-between gap-3">
<PageHeaderTitle className="text-xl">ROI Calculator</PageHeaderTitle>
<div className="flex min-w-0 flex-wrap items-center gap-2">
{actions}
{!readOnly && (
<Button size="sm" variant="outline" onClick={onConnect}>
<Link2 />
Connections
</Button>
)}
</div>
</PageHeader>
{notice}
<div className="flex flex-wrap items-center justify-between gap-2">
<Popover>
<PopoverTrigger render={<Button size="sm" variant="outline" className="self-start" />}>
{number(snapshot.repos.length)} {snapshot.repos.length === 1 ? "repository" : "repositories"}
<ChevronDown className="size-3.5" />
</PopoverTrigger>
<PopoverContent align="start" className="w-80 max-w-[calc(100vw-2rem)] gap-2 p-3">
<PopoverTitle>Repositories</PopoverTitle>
<ul className="max-h-64 space-y-2 overflow-y-auto text-xs break-words">
{snapshot.repos.map((repo) => (
<li key={repo}>{repo}</li>
))}
</ul>
</PopoverContent>
</Popover>
<div className="flex flex-wrap items-center gap-2">
<Select
value={String(days)}
disabled={!onPeriod || syncing}
onValueChange={(value) => {
if (value && Number(value) !== days) onPeriod?.(Number(value));
}}
items={rangeOptions.map((value) => ({ value: String(value), label: `Last ${value} days` }))}
>
<SelectTrigger size="sm" aria-label="Reporting period">
<SelectValue />
</SelectTrigger>
<SelectContent>
{rangeOptions.map((value) => (
<SelectItem key={value} value={String(value)}>
Last {value} days
</SelectItem>
))}
</SelectContent>
</Select>
<span className="flex h-8 items-center gap-2 text-xs text-muted-foreground">
<CalendarDays className="size-3.5 text-muted-foreground" />
{dateRange(current.window)}
</span>
<Select
value={comparison}
onValueChange={(value) => {
if (value) setComparison(value);
}}
items={[
{ value: "previous", label: "vs. previous period" },
{ value: "last_year", label: "vs. same period last year" },
]}
>
<SelectTrigger size="sm" aria-label="Comparison period">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="previous">vs. previous period</SelectItem>
<SelectItem value="last_year">vs. same period last year</SelectItem>
</SelectContent>
</Select>
</div>
</div>
<div className="grid grid-cols-2 gap-px overflow-hidden rounded-xl border bg-border lg:grid-cols-4">
<Metric
label={`Merged ${terms.plural}`}
value={number(current.merged_prs)}
current={current.merged_prs}
baseline={baseline.merged_prs}
detail={`${number(baseline.merged_prs)} in comparison`}
/>
<Metric
label="Median time to merge"
value={current.merged_prs === 0 ? "No merges" : duration(current.median_merge_hours)}
current={current.median_merge_hours ?? undefined}
baseline={baseline.median_merge_hours ?? undefined}
detail={`${duration(baseline.median_merge_hours)} in comparison`}
/>
<Metric
label="New bugs"
value={number(current.new_bug_labeled_issues)}
current={current.new_bug_labeled_issues}
baseline={baseline.new_bug_labeled_issues}
detail={`${number(baseline.new_bug_labeled_issues)} in comparison · bug-labeled issues`}
/>
<Metric
label={`Recorded spend / matched ${terms.singular}`}
value={money(cost)}
detail={
baselineCost === null
? "No gateway records for comparison"
: `${money(baselineCost)} in comparison · gateway only`
}
/>
</div>
<Tabs value={activeTab} onValueChange={setActiveTab} className="gap-3">
<div className="flex min-w-0 flex-wrap items-center gap-x-4 gap-y-2 border-b">
<PageTabsList className="min-w-0 flex-1 gap-4 border-0">
<PageTabsTrigger value="people">
Engineers <span className="ml-1.5 text-muted-foreground">{snapshot.people.length}</span>
</PageTabsTrigger>
<PageTabsTrigger value="pulls">{terms.requests}</PageTabsTrigger>
<PageTabsTrigger value="quality">Quality</PageTabsTrigger>
<PageTabsTrigger value="branches">Branch spend</PageTabsTrigger>
</PageTabsList>
{!readOnly && (
<Button
size="sm"
variant="outline"
aria-label="Link accounts"
title="Link accounts"
onClick={() => setAccountEmail("")}
>
<Users />
<span className="hidden sm:inline">Link accounts</span>
</Button>
)}
</div>
<TabsContent value="people">
<PeopleTable
snapshot={snapshot}
comparison={comparison}
onSelect={(selected) => setPersonEmail(selected.email)}
/>
</TabsContent>
<TabsContent value="pulls" className="space-y-5">
{current.merged_prs > 0 || baseline.merged_prs > 0 ? (
<div className="grid gap-4 lg:grid-cols-[1.6fr_1fr]">
<ShippingTrend snapshot={snapshot} comparison={comparison} />
<div className="flex flex-col justify-between rounded-xl border p-5">
<div>
<h2 className="text-sm font-medium">Behind the numbers</h2>
<p className="mt-3 text-sm leading-relaxed text-muted-foreground">
{number(current.agent_authored)} of {number(current.merged_prs)} {terms.plural} were authored by
agents or bots.
</p>
<p className="mt-3 text-xs leading-relaxed text-muted-foreground">
Human-authored median merge time:{" "}
<span className="font-medium text-foreground">
{duration(current.human_summary.median_merge_hours)}
</span>
, compared with {duration(baseline.human_summary.median_merge_hours)}.
</p>
</div>
<div className="mt-5 flex flex-wrap items-center justify-between gap-2 border-t pt-4">
<span className="text-xs text-muted-foreground">
{number(current.agents_without_requester)} agent {terms.plural} have no requester
</span>
</div>
</div>
</div>
) : (
<div className="rounded-xl border p-8 text-center">
<h2 className="text-base font-medium">No merged changes yet</h2>
<p className="mt-2 text-sm text-muted-foreground">
Your repositories are connected. New activity will appear after the next sync
</p>
</div>
)}
<p className="mb-4 text-xs text-muted-foreground">
All repository {terms.plural}, including agent work without a known requester
</p>
<PullList pulls={snapshot.pulls.current} provider={snapshot.source_provider} />
</TabsContent>
<TabsContent value="quality">
<Quality snapshot={snapshot} comparison={comparison} />
</TabsContent>
<TabsContent value="branches">
<BranchSpend snapshot={snapshot} />
</TabsContent>
</Tabs>
<div className="flex flex-wrap items-center justify-between gap-2 border-t pt-3 text-xs text-muted-foreground">
<span className="w-full">
{number(current.matched_internal_prs)} {terms.plural} matched to {snapshot.people.length} engineers ·{" "}
{number(current.agents_without_requester)} agent {terms.plural} without a requester
</span>
<span>Comparing with {dateRange(baseline.window)} · All dates UTC</span>
<span>Spend recorded by this gateway · Merge time is elapsed time, not effort</span>
</div>
{accountEmail !== null && (
<ObservedAccounts
accessToken={accessToken}
people={snapshot.people}
initialEmail={accountEmail}
onClose={() => setAccountEmail(null)}
onSaved={onRefresh}
/>
)}
{person && (
<PersonDetails
key={person.email}
person={person}
snapshot={snapshot}
comparison={comparison}
onClose={() => setPersonEmail(null)}
onEdit={
readOnly
? undefined
: () => {
setAccountEmail(person.email);
setPersonEmail(null);
}
}
/>
)}
</Page>
);
}

110
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.test.ts Normal file
View file

@ -0,0 +1,110 @@
import { describe, expect, it } from "vitest";
import {
change,
duration,
money,
visiblePeople,
weeklyMerges,
type ObservedPerson,
type ObservedSnapshot,
} from "./observedData";
const period = (merged: number, spend: number | null) => ({
merged_prs: merged,
prs_per_week: merged / 4,
median_merge_hours: null,
direct_authored: merged,
declared_agent_owned: 0,
gateway_recorded_spend: spend ?? 0,
recorded_spend_per_attributed_pr: spend !== null && merged > 0 ? spend / merged : null,
spend_observation: spend === null ? ("no_records" as const) : ("records_present" as const),
pr_urls: [],
});
const person = (name: string, merged: number, spend: number | null): ObservedPerson => ({
name,
email: `${name}@example.test`,
logins: [`old-${name}`],
periods: { current: period(merged, spend), previous: period(0, null), last_year: period(0, null) },
});
describe("observed ROI metrics", () => {
it("does not claim infinite growth when the baseline is missing", () => {
expect(change(12, 0)).toBeNull();
expect(change(0, 0)).toBeNull();
expect(change(15, 10)).toBe(50);
expect(change(0, 10)).toBe(-100);
});
it("distinguishes missing cost, measured zero, and small nonzero spend", () => {
expect(money(null)).toBe("Unavailable");
expect(money(0)).toBe("$0.00");
expect(money(0.001)).toBe("<$0.01");
expect(money(1.235)).toBe("$1.24");
expect(duration(null)).toBe("Unavailable");
});
it("searches historical identities and sorts without changing the source", () => {
const people = [person("Ari", 2, 8), person("Bea", 10, 0), person("Cam", 0, null)];
expect(visiblePeople(people, " OLD-ARI ", "merged").map((row) => row.name)).toEqual(["Ari"]);
expect(visiblePeople(people, "", "merged").map((row) => row.name)).toEqual(["Bea", "Ari", "Cam"]);
expect(visiblePeople(people, "", "cost").map((row) => row.name)).toEqual(["Ari", "Bea", "Cam"]);
expect(people.map((row) => row.name)).toEqual(["Ari", "Bea", "Cam"]);
expect(visiblePeople(people, "missing", "name")).toEqual([]);
});
it("keeps short elapsed merge times from rounding to zero hours", () => {
expect(duration(null)).toBe("Unavailable");
expect(duration(0)).toBe("0m");
expect(duration(16 / 3600)).toBe("<1m");
expect(duration(59 / 3600)).toBe("<1m");
expect(duration(1 / 60)).toBe("1m");
expect(duration(79 / 3600)).toBe("1.3m");
expect(duration(140 / 3600)).toBe("2.3m");
expect(duration(0.5)).toBe("30m");
expect(duration(1)).toBe("1h");
expect(duration(3.82)).toBe("3.8h");
});
it.each([7, 28, 90])("includes every day of a %s-day range in its weekly chart", (days) => {
const start = Date.parse("2026-01-01T00:00:00Z");
const atDay = (day: number) => new Date(start + day * 86400000).toISOString();
const pulls = Array.from({ length: days + 1 }, (_, day) => ({ merged_at: atDay(day) }));
const snapshot = {
periods: { current: { window: { start: "2026-01-01", end: atDay(days - 1).slice(0, 10) } } },
pulls: { current: pulls },
} as ObservedSnapshot;
const weeks = weeklyMerges(snapshot, "current");
expect(weeks).toHaveLength(Math.ceil(days / 7));
expect(weeks.reduce((sum, count) => sum + count, 0)).toBe(days);
expect(weeks.at(-1)).toBe(days % 7 || 7);
});
it("aligns comparisons to their own UTC windows and counts each boundary once", () => {
const currentStart = "2026-01-01";
const previousStart = "2025-12-04";
const pulls = [
"2026-01-01T00:00:00Z",
"2026-01-07T23:59:59Z",
"2026-01-08T00:00:00Z",
"2026-01-28T23:59:59Z",
"2026-01-29T00:00:00Z",
].map((merged_at, number) => ({
number,
merged_at,
title: "Example",
url: "https://github.com/example/repo/pull/1",
author: "ari",
agent: false,
merge_hours: 1,
}));
const snapshot = {
periods: {
current: { window: { start: currentStart, end: "2026-01-28" } },
previous: { window: { start: previousStart, end: "2025-12-31" } },
},
pulls: { current: pulls, previous: [{ ...pulls[0], merged_at: `${previousStart}T00:00:00Z` }] },
} as ObservedSnapshot;
expect(weeklyMerges(snapshot, "current")).toEqual([2, 1, 0, 1]);
expect(weeklyMerges(snapshot, "previous")).toEqual([1, 0, 0, 0]);
});
});

212
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.ts Normal file
View file

@ -0,0 +1,212 @@
import { z } from "zod";
const windowSchema = z.object({ start: z.string(), end: z.string() });
const personPeriodFields = {
merged_prs: z.number(),
prs_per_week: z.number(),
median_merge_hours: z.number().nullable(),
direct_authored: z.number(),
declared_agent_owned: z.number(),
gateway_recorded_spend: z.number(),
recorded_spend_per_attributed_pr: z.number().nullable(),
spend_observation: z.enum(["records_present", "no_records"]),
pr_urls: z.array(z.string()),
};
const personPeriodSchema = z.object(personPeriodFields);
const periodFields = {
window: windowSchema,
merged_prs: z.number(),
median_merge_hours: z.number().nullable(),
human_authored: z.number(),
agent_authored: z.number(),
missing_author: z.number(),
agents_without_requester: z.number(),
matched_internal_prs: z.number(),
new_bug_labeled_issues: z.number().nullable(),
new_regression_labeled_issues: z.number().nullable(),
explicitly_titled_revert_prs: z.number(),
matched_users_recorded_spend: z.number(),
spend_observation: z.enum(["records_present", "no_records"]),
human_summary: z.object({ median_merge_hours: z.number().nullable() }),
};
const periodSchema = z.object(periodFields);
const periods = <T extends z.ZodType>(schema: T) => z.object({ current: schema, previous: schema, last_year: schema });
const personFields = {
name: z.string(),
email: z.string(),
logins: z.array(z.string()),
periods: periods(personPeriodSchema),
accounts: z.array(z.object({ connection_id: z.string(), login: z.string() })).optional(),
};
const branchCostFields = {
repo: z.string(),
branch: z.string(),
spend: z.number().nullable(),
requests: z.number(),
status: z.enum(["matched", "unattributed", "ambiguous", "unavailable"]),
};
const branchSpendFields = { repo: z.string(), branch: z.string(), spend: z.number(), requests: z.number() };
const pullFields = {
connection_id: z.string().optional(),
number: z.number(),
title: z.string(),
url: z
.string()
.url()
.refine((url) => new URL(url).protocol === "https:"),
author: z.string(),
agent: z.boolean(),
merged_at: z.string(),
merge_hours: z.number().nullable(),
repo: z.string(),
source_repo: z.string(),
source_branch: z.string(),
branch_cost: z.object(branchCostFields),
};
const snapshotFields = {
source_provider: z.enum(["github", "gitlab", "mixed"]),
repos: z.array(z.string()),
unmatched_logins: z.array(z.string()),
unlinked_branches: z.array(z.object(branchSpendFields)),
captured_at: z.string(),
periods: periods(periodSchema),
people: z.array(z.object(personFields)),
pulls: periods(z.array(z.object(pullFields))),
};
export const observedSnapshotSchema = z.object(snapshotFields);
const settingsFields = {
id: z.string().optional(),
source_provider: z.enum(["github", "gitlab"]),
api_url: z.string(),
repos: z.array(z.string()),
has_token: z.boolean(),
connection_type: z.enum(["token", "app"]),
update_interval_minutes: z.number(),
ready: z.boolean(),
};
export const observedConnectionSchema = z.object(settingsFields);
export const observedSettingsSchema = z.object({
...settingsFields,
connections: z.array(observedConnectionSchema).optional(),
});
export type ObservedConnection = z.infer<typeof observedConnectionSchema>;
const statusFields = {
running: z.boolean(),
phase: z.string(),
stage: z.string(),
done: z.number(),
total: z.number(),
error: z.string().nullable(),
finished_at: z.string().nullable().optional(),
};
export const observedStatusSchema = z.object(statusFields);
export const observedReportResponseSchema = z.object({ report: observedSnapshotSchema.nullable() });
export type ObservedSettings = z.infer<typeof observedSettingsSchema>;
export type ObservedStatus = z.infer<typeof observedStatusSchema>;
export type ObservedSnapshot = z.infer<typeof observedSnapshotSchema>;
export type ObservedPerson = ObservedSnapshot["people"][number];
export type ObservedPull = ObservedSnapshot["pulls"]["current"][number];
export type Period = keyof ObservedSnapshot["periods"];
export type Comparison = Exclude<Period, "current">;
export type PeopleSort = "merged" | "spend" | "cost" | "name";
export const number = (value: number | null) =>
value === null ? "Unavailable" : value.toLocaleString("en-US", { maximumFractionDigits: 1 });
export function money(value: number | null) {
if (value === null) return "Unavailable";
if (value > 0 && value < 0.01) return "<$0.01";
return value.toLocaleString("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 2 });
}
export function duration(value: number | null) {
if (value === null) return "Unavailable";
if (value === 0) return "0m";
if (value < 1 / 60) return "<1m";
if (value < 1) return `${number(value * 60)}m`;
return `${number(value)}h`;
}
export const change = (current: number, baseline: number) =>
baseline === 0 ? null : ((current - baseline) / baseline) * 100;
export const accountLogins = (value: string) => [
...new Set(
value
.split(/[\s,]+/)
.map((login) => login.replace(/^@/, "").toLowerCase())
.filter(Boolean),
),
];
export const repositoryNames = (value: string) => [
...new Set(
value
.split(/[\s,]+/)
.filter(Boolean)
.map((repo) => {
const path = repo.replace(/^https:\/\/[^/]+\//, "");
return path.replace(/\/$/, "").replace(/\.git$/, "");
}),
),
];
export function dateRange(window: { start: string; end: string }) {
const date = (value: string) =>
new Date(`${value}T00:00:00Z`).toLocaleDateString("en-US", {
month: "short",
day: "numeric",
timeZone: "UTC",
});
return `${date(window.start)} – ${date(window.end)}, ${window.end.slice(0, 4)}`;
}
export function visiblePeople(people: ObservedPerson[], query: string, sort: PeopleSort) {
const value = (person: ObservedPerson) => {
const current = person.periods.current;
if (sort === "spend") return current.spend_observation === "no_records" ? -1 : current.gateway_recorded_spend;
if (sort === "cost") return current.recorded_spend_per_attributed_pr ?? -1;
return current.merged_prs;
};
return people
.filter((person) =>
[person.name, person.email, ...person.logins].join(" ").toLowerCase().includes(query.trim().toLowerCase()),
)
.toSorted((a, b) => (sort === "name" ? a.name.localeCompare(b.name) : value(b) - value(a)));
}
export function weeklyMerges(snapshot: ObservedSnapshot, period: Period) {
const start = Date.parse(`${snapshot.periods[period].window.start}T00:00:00Z`);
const end = Date.parse(`${snapshot.periods[period].window.end}T00:00:00Z`) + 86_400_000;
const days = (end - start) / 86_400_000;
return Array.from(
{ length: Math.ceil(days / 7) },
(_, week) =>
snapshot.pulls[period].filter((pull) => {
const day = (Date.parse(pull.merged_at) - start) / 86_400_000;
return day >= week * 7 && day < Math.min((week + 1) * 7, days);
}).length,
);
}
export function syncMessage(status: ObservedStatus, report: ObservedSnapshot | null) {
if (status.running) return status.total ? `${status.stage} · ${status.done} / ${status.total}` : status.stage;
return report ? `Updated ${new Date(report.captured_at).toLocaleString()}` : "";
}
export function recordedBranches(snapshot: ObservedSnapshot) {
const matched = snapshot.pulls.current.flatMap((pull) => {
const cost = pull.branch_cost;
if (cost.status !== "matched" || cost.spend === null) return [];
return [{ repo: cost.repo, branch: cost.branch, spend: cost.spend, requests: cost.requests }];
});
return [
...new Map([...matched, ...snapshot.unlinked_branches].map((row) => [`${row.repo}\n${row.branch}`, row])).values(),
].toSorted((a, b) => b.spend - a.spend);
}
export function changeTerms(provider: ObservedSnapshot["source_provider"]) {
if (provider === "mixed")
return { singular: "change", plural: "changes", requests: "Merged changes", lower: "merged changes" };
return provider === "gitlab"
? { singular: "MR", plural: "MRs", requests: "Merge requests", lower: "merge requests" }
: { singular: "PR", plural: "PRs", requests: "Pull requests", lower: "pull requests" };
}

42
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.test.ts Normal file
View file

@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import { observedSnapshotSchema, weeklyMerges, type Period } from "./observedData";
import { createObservedDemo } from "./observedDemo";
describe("observed sample report", () => {
it("clamps the year-ago comparison on leap day while keeping the selected length", () => {
const report = createObservedDemo(28, new Date("2024-03-01T12:00:00Z"));
expect(report.periods.current.window.end).toBe("2024-02-29");
expect(report.periods.last_year.window).toEqual({ start: "2023-02-01", end: "2023-02-28" });
});
it.each([7, 28, 90])("keeps totals, attribution, costs, and comparison windows consistent for %i days", (days) => {
const report = createObservedDemo(days, new Date("2026-10-03T14:00:00Z"));
expect(observedSnapshotSchema.safeParse(report).success).toBe(true);
for (const period of ["current", "previous", "last_year"] satisfies Period[]) {
const metrics = report.periods[period];
const pulls = report.pulls[period];
const people = report.people.map((person) => person.periods[period]);
const start = Date.parse(metrics.window.start);
const end = Date.parse(metrics.window.end) + 86_400_000;
expect((end - start) / 86_400_000).toBe(days);
expect(metrics.merged_prs).toBe(pulls.length);
expect(new Set(pulls.map((pull) => pull.url)).size).toBe(pulls.length);
expect(weeklyMerges(report, period).reduce((sum, value) => sum + value, 0)).toBe(pulls.length);
expect(pulls.every((pull) => Date.parse(pull.merged_at) >= start && Date.parse(pull.merged_at) < end)).toBe(true);
expect(metrics.matched_users_recorded_spend).toBe(
people.reduce((sum, person) => sum + person.gateway_recorded_spend, 0),
);
expect(people.reduce((sum, person) => sum + person.merged_prs, 0)).toBe(metrics.matched_internal_prs);
for (const person of people) {
const attributed = pulls.filter((pull) => person.pr_urls.includes(pull.url));
expect(attributed).toHaveLength(person.merged_prs);
expect(person.direct_authored).toBe(attributed.filter((pull) => !pull.agent).length);
expect(person.declared_agent_owned).toBe(attributed.filter((pull) => pull.agent).length);
expect(person.recorded_spend_per_attributed_pr).toBe(person.gateway_recorded_spend / person.merged_prs);
}
}
expect(Date.parse(report.periods.previous.window.end) + 86_400_000).toBe(
Date.parse(report.periods.current.window.start),
);
});
});

120
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.ts Normal file
View file

@ -0,0 +1,120 @@
import type { ObservedPerson, ObservedPull, ObservedSnapshot, Period } from "./observedData";
const DAY = 86_400_000;
const engineers = [
{ name: "Alex Rivera", login: "alex-demo", weekly: [8, 6, 4] },
{ name: "Sam Chen", login: "sam-demo", weekly: [6, 5, 4] },
{ name: "Jordan Lee", login: "jordan-demo", weekly: [4, 4, 3] },
];
const titles = [
"Add repository search",
"Fix retry handling",
"Speed up activity queries",
"Add usage export",
"Improve connection setup",
"Fix pagination",
];
const isoDate = (timestamp: number) => new Date(timestamp).toISOString().slice(0, 10);
function median(pulls: ObservedPull[]) {
const hours = pulls.map((pull) => pull.merge_hours ?? 0).toSorted((a, b) => a - b);
const middle = Math.floor(hours.length / 2);
if (!hours.length) return null;
return hours.length % 2 ? hours[middle] : (hours[middle - 1] + hours[middle]) / 2;
}
function samplePeriod(days: number, end: number, comparison: number) {
const start = end - (days - 1) * DAY;
const people = engineers.map((engineer, index) => {
const count = Math.round((engineer.weekly[comparison] * days) / 7);
const pulls: ObservedPull[] = Array.from({ length: count }, (_, position) => {
const number = 10000 * (comparison + 1) + index * 1000 + position;
const repo = index === 1 ? "demo/api" : "demo/web";
const branch = `feature/sample-${number}`;
const agent = position % 5 === 0;
return {
connection_id: index === 1 ? "demo-gitlab" : "demo-github",
number,
title: titles[position % titles.length],
url: `https://example.com/${repo}/changes/${number}`,
author: agent ? "demo-agent" : engineer.login,
agent,
merged_at: new Date(start + Math.floor((position * days) / count) * DAY + 12 * 3_600_000).toISOString(),
merge_hours: position === 0 ? 16 / 3600 : 4 + ((position * 7) % 24) + comparison * 6,
repo,
source_repo: repo,
source_branch: branch,
branch_cost: { repo, branch, spend: 2 + (position % 4), requests: 20 + position, status: "matched" },
};
});
const spend = pulls.reduce((total, pull) => total + (pull.branch_cost.spend ?? 0), 0);
const metrics: ObservedPerson["periods"]["current"] = {
merged_prs: pulls.length,
prs_per_week: (pulls.length * 7) / days,
median_merge_hours: median(pulls),
direct_authored: pulls.filter((pull) => !pull.agent).length,
declared_agent_owned: pulls.filter((pull) => pull.agent).length,
gateway_recorded_spend: spend,
recorded_spend_per_attributed_pr: pulls.length ? spend / pulls.length : null,
spend_observation: "records_present",
pr_urls: pulls.map((pull) => pull.url),
};
return { metrics, pulls };
});
const pulls = people.flatMap((person) => person.pulls).toSorted((a, b) => b.merged_at.localeCompare(a.merged_at));
const metrics: ObservedSnapshot["periods"]["current"] = {
window: { start: isoDate(start), end: isoDate(end) },
merged_prs: pulls.length,
median_merge_hours: median(pulls),
human_authored: pulls.filter((pull) => !pull.agent).length,
agent_authored: pulls.filter((pull) => pull.agent).length,
missing_author: 0,
agents_without_requester: 0,
matched_internal_prs: pulls.length,
new_bug_labeled_issues: Math.round(((comparison + 1) * days) / 7),
new_regression_labeled_issues: comparison,
explicitly_titled_revert_prs: 0,
matched_users_recorded_spend: people.reduce((total, person) => total + person.metrics.gateway_recorded_spend, 0),
spend_observation: "records_present",
human_summary: { median_merge_hours: median(pulls.filter((pull) => !pull.agent)) },
};
return { metrics, pulls, people };
}
export function createObservedDemo(days: number, now = new Date()): ObservedSnapshot {
const end = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - DAY;
const yearAgo = new Date(end);
const lastYear = yearAgo.getUTCFullYear() - 1;
const month = yearAgo.getUTCMonth();
const lastDay = new Date(Date.UTC(lastYear, month + 1, 0)).getUTCDate();
const lastYearEnd = Date.UTC(lastYear, month, Math.min(yearAgo.getUTCDate(), lastDay));
const periods = {
current: samplePeriod(days, end, 0),
previous: samplePeriod(days, end - days * DAY, 1),
last_year: samplePeriod(days, lastYearEnd, 2),
};
const personPeriod = (period: Period, index: number) => periods[period].people[index].metrics;
return {
source_provider: "mixed",
repos: ["demo/web", "demo/api"],
unmatched_logins: [],
unlinked_branches: [],
captured_at: now.toISOString(),
periods: {
current: periods.current.metrics,
previous: periods.previous.metrics,
last_year: periods.last_year.metrics,
},
people: engineers.map((engineer, index) => ({
name: engineer.name,
email: `${engineer.login}@example.com`,
logins: [engineer.login],
periods: {
current: personPeriod("current", index),
previous: personPeriod("previous", index),
last_year: personPeriod("last_year", index),
},
})),
pulls: { current: periods.current.pulls, previous: periods.previous.pulls, last_year: periods.last_year.pulls },
};
}

60
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/useObservedReport.ts Normal file
View file

@ -0,0 +1,60 @@
import { useCallback, useEffect, useState } from "react";
import { apiClient } from "@/components/networking";
import { extractProxyErrorMessage } from "@/lib/http/client";
import {
observedSettingsSchema,
observedReportResponseSchema,
observedStatusSchema,
type ObservedSettings,
type ObservedSnapshot,
type ObservedStatus,
} from "./observedData";
export type ObservedViewData = { settings: ObservedSettings; report: ObservedSnapshot | null; status: ObservedStatus };
export function useObservedReport(accessToken: string) {
const [data, setData] = useState<ObservedViewData | null>(null);
const [error, setError] = useState("");
const [revision, setRevision] = useState(0);
const refresh = useCallback(() => setRevision((value) => value + 1), []);
useEffect(() => {
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout>;
const options = { accessToken, signal: controller.signal };
async function poll(previous?: ObservedViewData) {
try {
const status = observedStatusSchema.parse(
await apiClient.get<unknown>("/roi-calculator/observed/sync", options),
);
const finished = previous?.status.running && !status.running;
const changed = !previous || previous.status.finished_at !== status.finished_at || finished;
const updated = changed
? await Promise.all([
apiClient
.get<unknown>("/roi-calculator/observed/settings", options)
.then((value) => observedSettingsSchema.parse(value)),
apiClient
.get<unknown>("/roi-calculator/observed/report", options)
.then((value) => observedReportResponseSchema.parse(value)),
])
: null;
if (controller.signal.aborted) return;
const existing = previous ? { ...previous, status } : null;
const next = updated ? { settings: updated[0], report: updated[1].report, status } : existing;
setData(next);
setError("");
timer = setTimeout(() => void poll(next ?? undefined), status.running ? 2000 : 30000);
} catch (reason) {
if (controller.signal.aborted) return;
setError(extractProxyErrorMessage(reason));
timer = setTimeout(() => void poll(previous), 10000);
}
}
void poll();
return () => {
controller.abort();
clearTimeout(timer);
};
}, [accessToken, revision]);
return { data, error, refresh };
}

7
ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/page.tsx
View file

@ -1,9 +1,10 @@
"use client";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import ROICalculatorView from "./_components/ROICalculatorView";
import ObservedROIView from "./_components/ObservedROIView";
export default function ROICalculatorPage() {
const { accessToken, userRole, isViewOnly } = useAuthorized();
return <ROICalculatorView accessToken={accessToken} userRole={userRole} isViewOnly={isViewOnly} />;
const { accessToken, isViewOnly } = useAuthorized();
if (!accessToken) return null;
return <ObservedROIView key={accessToken} accessToken={accessToken} isViewOnly={isViewOnly} />;
}

781
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -14331,6 +14331,129 @@ export interface paths {
patch?: never;
trace?: never;
};
"/roi-calculator/observed/apps": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Observed Apps */
get: operations["observed_apps_roi_calculator_observed_apps_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/observed/identities": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Observed Identities */
get: operations["get_observed_identities_roi_calculator_observed_identities_get"];
/** Save Observed Identities */
put: operations["save_observed_identities_roi_calculator_observed_identities_put"];
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/observed/oauth/{provider}/start": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/** Start Observed Authorization */
post: operations["start_observed_authorization_roi_calculator_observed_oauth__provider__start_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/observed/report": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Observed Report */
get: operations["get_observed_report_roi_calculator_observed_report_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/observed/repositories": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Observed Repositories */
get: operations["observed_repositories_roi_calculator_observed_repositories_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/observed/settings": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Observed Settings */
get: operations["get_observed_settings_roi_calculator_observed_settings_get"];
/** Save Observed Settings */
put: operations["save_observed_settings_roi_calculator_observed_settings_put"];
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/observed/sync": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Observed Sync */
get: operations["get_observed_sync_roi_calculator_observed_sync_get"];
put?: never;
/** Start Observed Sync */
post: operations["start_observed_sync_roi_calculator_observed_sync_post"];
/** Cancel Observed Sync */
delete: operations["cancel_observed_sync_roi_calculator_observed_sync_delete"];
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/roi-calculator/report": {
parameters: {
query?: never;
@ -39236,6 +39359,373 @@ export interface components {
[key: string]: unknown;
} | null;
};
/** ObservedAccount */
ObservedAccount: {
/** Connection Id */
connection_id: string;
/** Login */
login: string;
};
/** ObservedApp */
ObservedApp: {
/** Api Url */
api_url?: string | null;
/** Callback Url */
callback_url?: string | null;
/**
* Can Install
* @default false
*/
can_install: boolean;
/** Configured */
configured: boolean;
};
/** ObservedApps */
ObservedApps: {
github: components["schemas"]["ObservedApp"];
gitlab: components["schemas"]["ObservedApp"];
};
/** ObservedAuthorization */
ObservedAuthorization: {
/** Url */
url: string;
};
/** ObservedConnection */
ObservedConnection: {
/** Api Url */
api_url: string;
/**
* Connection Type
* @enum {string}
*/
connection_type: "token" | "app";
/** Has Token */
has_token: boolean;
/**
* Id
* @default
*/
id: string;
/** Ready */
ready: boolean;
/** Repos */
repos: string[];
/**
* Source Provider
* @enum {string}
*/
source_provider: "github" | "gitlab";
/** Update Interval Minutes */
update_interval_minutes: number;
};
/** ObservedConnectionIdentities */
ObservedConnectionIdentities: {
/** Api Url */
api_url: string;
/** Id */
id: string;
/** Identity Map */
identity_map: {
[key: string]: string;
};
/** Repos */
repos: string[];
/**
* Source Provider
* @enum {string}
*/
source_provider: "github" | "gitlab";
/** Unmatched Logins */
unmatched_logins: string[];
};
/** ObservedHumanSummary */
ObservedHumanSummary: {
/** Median Merge Hours */
median_merge_hours: number | null;
};
/** ObservedIdentities */
ObservedIdentities: {
/**
* Connections
* @default []
*/
connections: components["schemas"]["ObservedConnectionIdentities"][];
/** Gateway Emails */
gateway_emails: string[];
/** Identity Map */
identity_map: {
[key: string]: string;
};
/** Unmatched Logins */
unmatched_logins: string[];
};
/** ObservedIdentityUpdate */
ObservedIdentityUpdate: {
/** Accounts */
accounts?: components["schemas"]["ObservedAccount"][] | null;
/** Email */
email: string;
/**
* Logins
* @default []
*/
logins: string[];
};
/** ObservedPeriod */
ObservedPeriod: {
/** Agent Authored */
agent_authored: number;
/** Agents Without Requester */
agents_without_requester: number;
/** Explicitly Titled Revert Prs */
explicitly_titled_revert_prs: number;
/** Human Authored */
human_authored: number;
human_summary: components["schemas"]["ObservedHumanSummary"];
/** Matched Internal Prs */
matched_internal_prs: number;
/** Matched Users Recorded Spend */
matched_users_recorded_spend: number;
/** Median Merge Hours */
median_merge_hours: number | null;
/** Merged Prs */
merged_prs: number;
/** Missing Author */
missing_author: number;
/** New Bug Labeled Issues */
new_bug_labeled_issues: number | null;
/** New Regression Labeled Issues */
new_regression_labeled_issues: number | null;
/**
* Spend Observation
* @enum {string}
*/
spend_observation: "records_present" | "no_records";
window: components["schemas"]["ObservedWindow"];
};
/** ObservedPeriods */
ObservedPeriods: {
current: components["schemas"]["ObservedPeriod"];
last_year: components["schemas"]["ObservedPeriod"];
previous: components["schemas"]["ObservedPeriod"];
};
/** ObservedPerson */
ObservedPerson: {
/**
* Accounts
* @default []
*/
accounts: components["schemas"]["ObservedAccount"][];
/** Email */
email: string;
/** Logins */
logins: string[];
/** Name */
name: string;
periods: components["schemas"]["ObservedPersonPeriods"];
};
/** ObservedPersonPeriod */
ObservedPersonPeriod: {
/** Declared Agent Owned */
declared_agent_owned: number;
/** Direct Authored */
direct_authored: number;
/** Gateway Recorded Spend */
gateway_recorded_spend: number;
/** Median Merge Hours */
median_merge_hours: number | null;
/** Merged Prs */
merged_prs: number;
/** Pr Urls */
pr_urls: string[];
/** Prs Per Week */
prs_per_week: number;
/** Recorded Spend Per Attributed Pr */
recorded_spend_per_attributed_pr: number | null;
/**
* Spend Observation
* @enum {string}
*/
spend_observation: "records_present" | "no_records";
};
/** ObservedPersonPeriods */
ObservedPersonPeriods: {
current: components["schemas"]["ObservedPersonPeriod"];
last_year: components["schemas"]["ObservedPersonPeriod"];
previous: components["schemas"]["ObservedPersonPeriod"];
};
/** ObservedPullPeriods */
ObservedPullPeriods: {
/** Current */
current: components["schemas"]["ObservedPullResponse"][];
/** Last Year */
last_year: components["schemas"]["ObservedPullResponse"][];
/** Previous */
previous: components["schemas"]["ObservedPullResponse"][];
};
/** ObservedPullResponse */
ObservedPullResponse: {
/**
* Agent
* @default false
*/
agent: boolean;
/** Author */
author: string;
branch_cost: components["schemas"]["ROIBranchAttribution"];
/**
* Connection Id
* @default
*/
connection_id: string;
/** Created At */
created_at?: string | null;
/** Merge Hours */
merge_hours: number | null;
/**
* Merged At
* Format: date-time
*/
merged_at: string;
/** Number */
number: number;
/**
* Profile Email
* @default
*/
profile_email: string;
/** Repo */
repo: string;
/**
* Requester
* @default
*/
requester: string;
/**
* Source Branch
* @default
*/
source_branch: string;
/**
* Source Repo
* @default
*/
source_repo: string;
/** Title */
title: string;
/** Url */
url: string;
};
/** ObservedReport */
ObservedReport: {
/**
* Captured At
* Format: date-time
*/
captured_at: string;
/**
* Connections
* @default []
*/
connections: components["schemas"]["ObservedSource"][];
/** People */
people: components["schemas"]["ObservedPerson"][];
periods: components["schemas"]["ObservedPeriods"];
pulls: components["schemas"]["ObservedPullPeriods"];
/** Repos */
repos: string[];
/**
* Source Provider
* @enum {string}
*/
source_provider: "github" | "gitlab" | "mixed";
/** Unlinked Branches */
unlinked_branches: components["schemas"]["ROIBranchSpend"][];
/** Unmatched Logins */
unmatched_logins: string[];
};
/** ObservedReportResponse */
ObservedReportResponse: {
report: components["schemas"]["ObservedReport"] | null;
};
/** ObservedSettings */
ObservedSettings: {
/** Api Url */
api_url: string;
/**
* Connection Type
* @enum {string}
*/
connection_type: "token" | "app";
/**
* Connections
* @default []
*/
connections: components["schemas"]["ObservedConnection"][];
/** Has Token */
has_token: boolean;
/**
* Id
* @default
*/
id: string;
/** Ready */
ready: boolean;
/** Repos */
repos: string[];
/**
* Source Provider
* @enum {string}
*/
source_provider: "github" | "gitlab";
/** Update Interval Minutes */
update_interval_minutes: number;
};
/** ObservedSettingsUpdate */
ObservedSettingsUpdate: {
/** Api Url */
api_url: string;
/** Connection Id */
connection_id?: string | null;
/** Repos */
repos: string[];
/**
* Source Provider
* @enum {string}
*/
source_provider: "github" | "gitlab";
/** Token */
token?: string | null;
/** Update Interval Minutes */
update_interval_minutes?: number | null;
};
/** ObservedSource */
ObservedSource: {
/** Api Url */
api_url: string;
/** Id */
id: string;
/** Repos */
repos: string[];
/**
* Source Provider
* @enum {string}
*/
source_provider: "github" | "gitlab";
};
/** ObservedWindow */
ObservedWindow: {
/**
* End
* Format: date
*/
end: string;
/**
* Start
* Format: date
*/
start: string;
};
/**
* OpenIdConnectSecurityScheme
* @description Defines a security scheme using OpenID Connect.
@ -41591,6 +42081,12 @@ export interface components {
};
/** Ready */
ready: boolean;
/**
* Report Mode
* @default legacy
* @enum {string}
*/
report_mode: "legacy" | "observed";
/** Repos */
repos: string[];
/**
@ -41620,6 +42116,8 @@ export interface components {
gitlab_api_url?: string | null;
/** Gitlab Token */
gitlab_token?: string | null;
/** Report Mode */
report_mode?: ("legacy" | "observed") | null;
/** Repos */
repos?: string[] | null;
/** Source Provider */
@ -69452,6 +69950,289 @@ export interface operations {
};
};
};
observed_apps_roi_calculator_observed_apps_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedApps"];
};
};
};
};
get_observed_identities_roi_calculator_observed_identities_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedIdentities"];
};
};
};
};
save_observed_identities_roi_calculator_observed_identities_put: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["ObservedIdentityUpdate"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedReportResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
start_observed_authorization_roi_calculator_observed_oauth__provider__start_post: {
parameters: {
query?: {
install?: boolean;
};
header?: never;
path: {
provider: "github" | "gitlab";
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedAuthorization"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_observed_report_roi_calculator_observed_report_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedReportResponse"];
};
};
};
};
observed_repositories_roi_calculator_observed_repositories_get: {
parameters: {
query?: {
connection?: string | null;
query?: string;
page?: number;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ROIRepositoriesResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_observed_settings_roi_calculator_observed_settings_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedSettings"];
};
};
};
};
save_observed_settings_roi_calculator_observed_settings_put: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["ObservedSettingsUpdate"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ObservedSettings"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_observed_sync_roi_calculator_observed_sync_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ROISyncStatus"];
};
};
};
};
start_observed_sync_roi_calculator_observed_sync_post: {
parameters: {
query?: {
days?: number | null;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
202: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ROISyncStatus"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
cancel_observed_sync_roi_calculator_observed_sync_delete: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ROISyncStatus"];
};
};
};
};
get_roi_calculator_report_roi_calculator_report_get: {
parameters: {
query?: {